From 7815ed417c46dbc8088ac76c830b095beb9cf948 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 03:16:28 +0000 Subject: [PATCH] Wider artist pool (artists credited on DR-played tracks, groups; weekly refresh, oldest first); Platt recalibration of the add model on recent closed windows; Bauer plays per artist on the site Co-Authored-By: Claude Opus 5.5 --- src/dr.rs | 45 ++++++++++++++++++++++++++++++++++++-------- src/main.rs | 2 +- src/model/adds.rs | 47 +++++++++++++++++++++++++++++++++++++++++++--- src/web.rs | 39 +++++++++++++++++++++++++++++++++++++- src/web/index.html | 15 ++++++++++----- 5 files changed, 130 insertions(+), 18 deletions(-) diff --git a/src/dr.rs b/src/dr.rs index d1ff2d4..72f9ac1 100644 --- a/src/dr.rs +++ b/src/dr.rs @@ -92,18 +92,47 @@ pub fn collect_plays(db: &Db, http: &Http, days: i64, log: &mut Vec) -> /// Artist pages: 52 weeks of weekly plays per channel. Fetched for every artist seen in the /// plays, again once the page is `max_age_days` old. `limit` caps the number per run. pub fn collect_artists(db: &Db, http: &Http, max_age_days: i64, limit: usize, log: &mut Vec) -> Result { - let cutoff = (Utc::now() - Duration::days(max_age_days)).to_rfc3339(); - let mut ids: Vec = Vec::new(); + // Where artist ids come from: DR's play logs, and (from stored artist pages) artists credited + // on tracks DR has played and groups an artist belongs to ("Medlem af"). Band members alone + // aren't release candidates and are left out. + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS dr_artist_sources ( + artist_id TEXT NOT NULL, source TEXT NOT NULL, first_seen TEXT NOT NULL, + PRIMARY KEY (artist_id, source) + );", + )?; + let now = Utc::now().to_rfc3339(); { + let mut add = db.conn.prepare("INSERT OR IGNORE INTO dr_artist_sources VALUES (?1, ?2, ?3)")?; let mut st = db.conn.prepare("SELECT DISTINCT artist_ids FROM plays WHERE source = 'dr' AND artist_ids != ''")?; - let rows = st.query_map([], |r| r.get::<_, String>(0))?; - let mut seen = std::collections::BTreeSet::new(); - for r in rows { for id in r?.split(',') { if !id.is_empty() { seen.insert(id.to_string()); } } } - let mut fresh = db.conn.prepare("SELECT count(*) FROM dr_artists WHERE artist_id = ?1 AND fetched_at > ?2")?; - for id in seen { - if fresh.query_row(params![id, cutoff], |r| r.get::<_, i64>(0))? == 0 { ids.push(id); } + let rows: Vec = st.query_map([], |r| r.get(0))?.collect::>()?; + for r in rows { for id in r.split(',') { if !id.is_empty() { add.execute(params![id, "plays", now])?; } } } + let mut st = db.conn.prepare("SELECT raw FROM dr_artists")?; + let raws: Vec = st.query_map([], |r| r.get(0))?.collect::>()?; + for raw in raws { + let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); + for rel in v["artist"]["relationships"].as_array().into_iter().flatten() { + if rel["type"].as_str() == Some("Medlem af") { + if let Some(id) = rel["artist"]["id"].as_str() { add.execute(params![id, "group", now])?; } + } + } + for list in ["mostPlayed", "latestPlayed"] { + for t in v[list].as_array().into_iter().flatten() { + for role in t["roles"].as_array().into_iter().flatten() { + if let Some(id) = role["artist"]["id"].as_str() { add.execute(params![id, "credited", now])?; } + } + } + } } } + // Never read first, then the stalest; everything older than max_age_days. + let cutoff = (Utc::now() - Duration::days(max_age_days)).to_rfc3339(); + let ids: Vec = db.conn.prepare( + "SELECT s.artist_id FROM (SELECT DISTINCT artist_id FROM dr_artist_sources) s + LEFT JOIN dr_artists a ON a.artist_id = s.artist_id + WHERE a.fetched_at IS NULL OR a.fetched_at < ?1 + ORDER BY a.fetched_at IS NOT NULL, a.fetched_at")? + .query_map([cutoff], |r| r.get(0))?.collect::>()?; let mut done = 0; for id in ids.into_iter().take(limit) { let props = match page_props(http, &format!("/lyd/musik/kunstner/{id}")) { diff --git a/src/main.rs b/src/main.rs index 34e0546..5196284 100644 --- a/src/main.rs +++ b/src/main.rs @@ -86,7 +86,7 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let dir = std::path::Path::new(db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations"); let r = dr::collect_rotation_pdf(db, &http, &dir); step("dr rotation list", r.map(|f| f.map(|f| format!("saved {f}")).unwrap_or_else(|| "unchanged".into())), &mut log); - let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 400 }, &mut log); + let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 1500 }, &mut log); step("dr artist pages", r.map(|n| format!("{n} fetched")), &mut log); } if wants("bauer") { diff --git a/src/model/adds.rs b/src/model/adds.rs index 1f398d4..a5ff75a 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -126,15 +126,56 @@ pub struct AddModel { pub h: usize, pub channel: Vec, pub any: Vec, + /// Platt recalibration (a, b): p' = sigmoid(a + b * logit(p)); identity when there wasn't + /// enough history to fit it honestly. + pub platt_channel: (f64, f64), + pub platt_any: (f64, f64), } +/// Recent origins used to fit the recalibration. +const CAL_ORIGINS: usize = 6; + +fn logit(p: f64) -> f64 { let p = p.clamp(1e-6, 1.0 - 1e-6); (p / (1.0 - p)).ln() } +fn platt(ab: (f64, f64), p: f64) -> f64 { glm::sigmoid(ab.0 + ab.1 * logit(p)).clamp(1e-6, 1.0 - 1e-6) } + fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive { let hi = last_week.saturating_sub(h + W); LAGS - 1..=hi } impl AddModel { + /// The logits on every window closed by `last_week`, then a recalibration: an inner model fitted + /// on windows closed before the last CAL_ORIGINS origins predicts those origins, and a Platt fit + /// maps its probabilities to what happened. (The raw fits were overconfident above ~0.4.) pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel { + let mut m = Self::fit_raw(p, r, h, last_week); + let hi = last_week.saturating_sub(h + W); + if hi + 1 >= CAL_ORIGINS + LAGS { + let c0 = hi + 1 - CAL_ORIGINS; + // the inner model may only use windows that closed before the first calibration origin + if c0 >= LAGS - 1 + h + W + 4 { + let inner = Self::fit_raw(p, r, h, c0); + let (mut dc, mut da) = (Design::new(2), Design::new(2)); + for t in c0..=hi { + for a in 0..p.a() { + let (pc, pa) = inner.predict(p, r, a, t, None); + for c in 0..C { + if pc[c] < 1e-4 && !r.add_in_window(a, c, t, h) { continue; } // certain zeros carry no information + dc.push(&[1.0, logit(pc[c])], r.add_in_window(a, c, t, h) as i32 as f64, 1.0); + } + da.push(&[1.0, logit(pa)], r.add_any_in_window(a, t, h) as i32 as f64, 1.0); + } + } + let (bc, ba) = (glm::logit(&dc, 0.0), glm::logit(&da, 0.0)); + // keep the map monotone and sane + if bc[1] > 0.2 { m.platt_channel = (bc[0], bc[1]); } + if ba[1] > 0.2 { m.platt_any = (ba[0], ba[1]); } + } + } + m + } + + fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel { let (mut dc, mut da) = (Design::new(PC), Design::new(PA)); let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]); for t in origins(h, last_week) { @@ -150,7 +191,7 @@ impl AddModel { da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0); } } - AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0) } + AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0) } } /// (P(add) per channel, P(add on any channel)) from origin t. @@ -160,12 +201,12 @@ impl AddModel { for c in 0..C { row_channel(p, r, a, c, t, self.h, &mut x); if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); } - pc[c] = glm::sigmoid(glm::dot(&self.channel, &x)).clamp(1e-6, 1.0 - 1e-6); + pc[c] = platt(self.platt_channel, glm::sigmoid(glm::dot(&self.channel, &x))); } let mut xa = vec![0.0; PA]; row_any(p, r, a, t, self.h, &mut xa); if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); } - (pc, glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6)) + (pc, platt(self.platt_any, glm::sigmoid(glm::dot(&self.any, &xa)))) } } diff --git a/src/web.rs b/src/web.rs index ac19a27..3cf0576 100644 --- a/src/web.rs +++ b/src/web.rs @@ -191,6 +191,34 @@ fn fold(e: &mut Value, target: &str, ch: &str, prob: Option, mean: Option Result)>>> { + let mut out: HashMap)>> = HashMap::new(); + let mut st = c.prepare( + "SELECT station, artist, title, count(*) FROM plays WHERE source = 'bauer' + AND played_at > (SELECT strftime('%Y-%m-%dT%H:%M:%SZ', max(played_at), '-7 days') FROM plays WHERE source = 'bauer') + GROUP BY station, artist, title")?; + let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?; + for r in rows { + let (station, artist, title, n) = r?; + let mut parts = vec![artist.to_lowercase()]; + for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " vs "] { + parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::>()).collect(); + } + let mut keys: Vec = parts.iter().map(|p| crate::model::panel::key(p)).collect(); + keys.push(crate::model::panel::key(&artist)); + keys.sort(); keys.dedup(); + for k in keys.into_iter().filter(|k| !k.is_empty()) { + let e = out.entry(k).or_default().entry(station.clone()).or_insert((0, Vec::new())); + e.0 += n; + e.1.push((title.clone(), n)); + } + } + Ok(out) +} + fn rank(st: &State, h: usize) -> Result { let c = ro(&st.db_path)?; let Some(made) = latest_run(&c)? else { return ok(json!({"rows": []})) }; @@ -217,6 +245,11 @@ fn rank(st: &State, h: usize) -> Result { if target == "add_any" && window.is_empty() { window = week; } fold(e, &target, ch, prob, mean, lo, hi); } + let bauer = bauer_week(&c)?; + for e in rows.values_mut() { + let k = crate::model::panel::key(e["name"].as_str().unwrap_or("")); + if let Some(b) = bauer.get(&k) { e["bauer"] = json!(b.iter().map(|(s, v)| (s.clone(), v.0)).collect::>()); } + } let mut out: Vec = rows.into_values().collect(); out.sort_by(|a, b| b["add_any"].as_f64().unwrap_or(0.0).total_cmp(&a["add_any"].as_f64().unwrap_or(0.0))); let body = Arc::new(json!({"made_at": made, "h": h, "window_start": window, "rows": out}).to_string()); @@ -288,8 +321,12 @@ fn artist(st: &State, id: &str) -> Result { } } + let bauer: Value = bauer_week(&c)?.remove(&key).map(|b| json!(b.into_iter().map(|(s, (n, mut songs))| { + songs.sort_by(|x, y| y.1.cmp(&x.1)); + (s, json!({"plays": n, "songs": songs.into_iter().take(5).map(|(t, n)| json!({"title": t, "plays": n})).collect::>()})) + }).collect::>())).unwrap_or(Value::Null); ok(json!({ - "id": id, "name": name, "country": a["country"], "gender": a["gender"], "description": a["description"], + "id": id, "name": name, "bauer": bauer, "country": a["country"], "gender": a["gender"], "description": a["description"], "debut": a["lifespan"]["beginYear"], "type": a["type"], "image": image, "top_tracks": top, "weeks": weeks, "plays": plays, "forecasts": fc, "now": now_out, "charts": charts, })) diff --git a/src/web/index.html b/src/web/index.html index e4a30fe..490ab01 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -139,7 +139,7 @@ svg text { fill: var(--muted); font-size: 11px; } -

New add = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.

+

"In rotation now" lists DR stations where the artist is on the list, then Bauer stations (grey) with 14+ plays in the last 7 days. New add = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.