Hitlisten: Track Top-40 (since 2007) and Album Top-40 (since 2001) beside the Airplay Top-20, with weeks at #1 and Spotify ids; one hitlisten_charts table (hitlisten_airplay stays as a view)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+10
-7
@@ -5,7 +5,7 @@
|
|||||||
//! DJ segues; checked by hand on P3: different songs, not duplicates)
|
//! DJ segues; checked by hand on P3: different songs, not duplicates)
|
||||||
//! - DR cross-check: plays we counted per artist, channel and week against the weekly totals on
|
//! - DR cross-check: plays we counted per artist, channel and week against the weekly totals on
|
||||||
//! DR's artist pages (two independent DR sources), for weeks we cover completely
|
//! DR's artist pages (two independent DR sources), for weeks we cover completely
|
||||||
//! - Hitlisten: weeks missing, and weeks without exactly 20 rows
|
//! - Hitlisten (airplay, Track and Album Top-40): weeks missing, weeks without a full chart
|
||||||
|
|
||||||
use crate::{db::Db, hitlisten};
|
use crate::{db::Db, hitlisten};
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
@@ -102,12 +102,14 @@ fn dr_cross_check(db: &Db) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn hitlisten_check(db: &Db) -> Result<()> {
|
fn hitlisten_check(db: &Db) -> Result<()> {
|
||||||
println!("\n== Hitlisten Airplay Top-20");
|
|
||||||
hitlisten::create_tables(db)?;
|
hitlisten::create_tables(db)?;
|
||||||
let weeks: Vec<(i64, i64, i64)> = db.conn.prepare("SELECT year, week, count(*) FROM hitlisten_airplay GROUP BY year, week ORDER BY year, week")?
|
for &(chart, size) in hitlisten::CHARTS {
|
||||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
|
println!("\n== Hitlisten {chart}");
|
||||||
if weeks.is_empty() { println!("nothing collected"); return Ok(()); }
|
let weeks: Vec<(i64, i64, i64)> = db.conn.prepare(
|
||||||
let odd: Vec<String> = weeks.iter().filter(|w| w.2 != 20).map(|w| format!("{}-W{:02} ({} rows)", w.0, w.1, w.2)).collect();
|
"SELECT year, week, count(*) FROM hitlisten_charts WHERE chart = ?1 GROUP BY year, week ORDER BY year, week")?
|
||||||
|
.query_map([chart], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
|
||||||
|
if weeks.is_empty() { println!("nothing collected"); continue; }
|
||||||
|
let odd: Vec<String> = weeks.iter().filter(|w| w.2 as usize != size).map(|w| format!("{}-W{:02} ({} rows)", w.0, w.1, w.2)).collect();
|
||||||
// Missing weeks between the first and the last (ISO weeks; some years have 53).
|
// Missing weeks between the first and the last (ISO weeks; some years have 53).
|
||||||
let have: std::collections::HashSet<(i64, i64)> = weeks.iter().map(|w| (w.0, w.1)).collect();
|
let have: std::collections::HashSet<(i64, i64)> = weeks.iter().map(|w| (w.0, w.1)).collect();
|
||||||
let mut missing = Vec::new();
|
let mut missing = Vec::new();
|
||||||
@@ -119,9 +121,10 @@ fn hitlisten_check(db: &Db) -> Result<()> {
|
|||||||
if !have.contains(&(y, w)) { missing.push(format!("{y}-W{w:02}")); }
|
if !have.contains(&(y, w)) { missing.push(format!("{y}-W{w:02}")); }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println!("{} weeks from {fy}-W{fw:02} to {ly}-W{lw:02}; {} missing; {} without 20 rows",
|
println!("{} weeks from {fy}-W{fw:02} to {ly}-W{lw:02}; {} missing; {} without {size} rows",
|
||||||
weeks.len(), missing.len(), odd.len());
|
weeks.len(), missing.len(), odd.len());
|
||||||
if !missing.is_empty() { println!("missing: {}", missing.iter().take(40).cloned().collect::<Vec<_>>().join(", ")); }
|
if !missing.is_empty() { println!("missing: {}", missing.iter().take(40).cloned().collect::<Vec<_>>().join(", ")); }
|
||||||
if !odd.is_empty() { println!("odd: {}", odd.iter().take(20).cloned().collect::<Vec<_>>().join(", ")); }
|
if !odd.is_empty() { println!("odd: {}", odd.iter().take(20).cloned().collect::<Vec<_>>().join(", ")); }
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -84,7 +84,7 @@ pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Res
|
|||||||
create_tables(db)?;
|
create_tables(db)?;
|
||||||
crate::hitlisten::create_tables(db)?;
|
crate::hitlisten::create_tables(db)?;
|
||||||
let todo: Vec<(String, String)> = db.conn.prepare(
|
let todo: Vec<(String, String)> = db.conn.prepare(
|
||||||
"SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_airplay) s
|
"SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_charts WHERE chart != 'a40') s
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title)
|
WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title)
|
||||||
AND artist != '' AND title != ''")?
|
AND artist != '' AND title != ''")?
|
||||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
||||||
|
|||||||
+68
-29
@@ -1,7 +1,11 @@
|
|||||||
//! Hitlisten (IFPI Danmark's official charts): the weekly Airplay Top-20, the most-played songs
|
//! Hitlisten (IFPI Danmark's official charts), every week of the archive:
|
||||||
//! on the biggest Danish radio stations (monitored by M&I Service), every week since week 38 of 2001.
|
|
||||||
//!
|
//!
|
||||||
//! https://www.hitlisten.nu/default.asp?w={week}&y={year}&list=airplay
|
//! airplay Airplay Top-20 since 2001-W38 the most-played songs on the biggest Danish radio
|
||||||
|
//! stations (monitored by M&I Service)
|
||||||
|
//! t40 Track Top-40 since 2007-W01 sales + streaming (streaming counted since Nov 2014)
|
||||||
|
//! a40 Album Top-40 since 2001
|
||||||
|
//!
|
||||||
|
//! https://www.hitlisten.nu/default.asp?w={week}&y={year}&list={chart}
|
||||||
//!
|
//!
|
||||||
//! The page's week selector lists every week that exists; weeks already stored aren't fetched again.
|
//! The page's week selector lists every week that exists; weeks already stored aren't fetched again.
|
||||||
|
|
||||||
@@ -12,16 +16,38 @@ use rusqlite::params;
|
|||||||
|
|
||||||
const BASE: &str = "https://www.hitlisten.nu/default.asp";
|
const BASE: &str = "https://www.hitlisten.nu/default.asp";
|
||||||
|
|
||||||
|
/// (chart code, rows per week)
|
||||||
|
pub const CHARTS: &[(&str, usize)] = &[("airplay", 20), ("t40", 40), ("a40", 40)];
|
||||||
|
|
||||||
pub fn create_tables(db: &Db) -> Result<()> {
|
pub fn create_tables(db: &Db) -> Result<()> {
|
||||||
db.conn.execute_batch(
|
db.conn.execute_batch(
|
||||||
"CREATE TABLE IF NOT EXISTS hitlisten_airplay (
|
"CREATE TABLE IF NOT EXISTS hitlisten_charts (
|
||||||
|
chart TEXT NOT NULL, -- airplay | t40 | a40
|
||||||
year INTEGER NOT NULL, week INTEGER NOT NULL, position INTEGER NOT NULL,
|
year INTEGER NOT NULL, week INTEGER NOT NULL, position INTEGER NOT NULL,
|
||||||
last_week TEXT, -- last week's position, 'NY' (new), 'RE' (re-entry) or empty
|
last_week TEXT, -- last week's position, 'NY' (new), 'RE' (re-entry) or empty
|
||||||
weeks_on_chart INTEGER, peak INTEGER,
|
weeks_on_chart INTEGER, peak INTEGER, weeks_at_1 INTEGER,
|
||||||
artist TEXT NOT NULL, title TEXT NOT NULL, label TEXT,
|
artist TEXT NOT NULL, title TEXT NOT NULL, label TEXT,
|
||||||
PRIMARY KEY (year, week, position)
|
spotify_id TEXT, -- t40/a40: the release's Spotify track or album id, when linked
|
||||||
|
PRIMARY KEY (chart, year, week, position)
|
||||||
);",
|
);",
|
||||||
)?;
|
)?;
|
||||||
|
// The first version stored the airplay chart on its own; move it in, keep the old name as a view.
|
||||||
|
let old_table: i64 = db.conn.query_row(
|
||||||
|
"SELECT count(*) FROM sqlite_master WHERE name = 'hitlisten_airplay' AND type = 'table'", [], |r| r.get(0))?;
|
||||||
|
if old_table > 0 {
|
||||||
|
db.conn.execute_batch(
|
||||||
|
"BEGIN;
|
||||||
|
INSERT OR IGNORE INTO hitlisten_charts (chart, year, week, position, last_week, weeks_on_chart, peak, artist, title, label)
|
||||||
|
SELECT 'airplay', year, week, position, last_week, weeks_on_chart, peak, artist, title, label FROM hitlisten_airplay;
|
||||||
|
DROP TABLE hitlisten_airplay;
|
||||||
|
COMMIT;",
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
db.conn.execute_batch(
|
||||||
|
"CREATE VIEW IF NOT EXISTS hitlisten_airplay AS
|
||||||
|
SELECT year, week, position, last_week, weeks_on_chart, peak, artist, title, label
|
||||||
|
FROM hitlisten_charts WHERE chart = 'airplay';",
|
||||||
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,25 +58,31 @@ fn text(s: &str) -> String {
|
|||||||
|
|
||||||
pub struct Row {
|
pub struct Row {
|
||||||
pub position: i64,
|
pub position: i64,
|
||||||
pub last_week: String, // last week's position, "NY" (new), "RE" (re-entry) or empty
|
pub last_week: String,
|
||||||
pub weeks_on_chart: Option<i64>,
|
pub weeks_on_chart: Option<i64>,
|
||||||
pub peak: Option<i64>,
|
pub peak: Option<i64>,
|
||||||
|
pub weeks_at_1: Option<i64>,
|
||||||
pub artist: String,
|
pub artist: String,
|
||||||
pub title: String,
|
pub title: String,
|
||||||
pub label: String,
|
pub label: String,
|
||||||
|
pub spotify_id: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The chart rows of one week's page. Each row is a <div id="liniesmall"> block; the fields
|
/// The chart rows of one week's page. The airplay chart's rows are <div id="liniesmall">
|
||||||
/// are read one by one so markup variants (re-entries, bold NY/RE) don't drop rows.
|
/// blocks, the Top-40s' are <div id="linien">; the fields are read one by one so markup
|
||||||
|
/// variants (re-entries, bold NY/RE, links around the label) don't drop rows.
|
||||||
pub fn parse(html: &str) -> Vec<Row> {
|
pub fn parse(html: &str) -> Vec<Row> {
|
||||||
let tags = Regex::new(r"<[^>]+>").unwrap();
|
let tags = Regex::new(r"(?s)<!--.*?-->|<[^>]+>").unwrap();
|
||||||
let field = |block: &str, re: &str| -> String {
|
let field = |block: &str, re: &str| -> String {
|
||||||
Regex::new(re).unwrap().captures(block)
|
Regex::new(re).unwrap().captures(block)
|
||||||
.map(|c| text(&tags.replace_all(&c[1], " "))).unwrap_or_default()
|
.map(|c| text(&tags.replace_all(&c[1], " "))).unwrap_or_default()
|
||||||
};
|
};
|
||||||
html.split(r#"<div id="liniesmall">"#)
|
let spotify = Regex::new(r"open\.spotify\.com/(?:track|album)/([A-Za-z0-9]{22})").unwrap();
|
||||||
.skip(1)
|
let block_start = Regex::new(r#"<div id="linien(?:small)?">|<div id="liniesmall">"#).unwrap();
|
||||||
.filter_map(|b| {
|
let starts: Vec<usize> = block_start.find_iter(html).map(|m| m.start()).collect();
|
||||||
|
starts.iter().enumerate()
|
||||||
|
.filter_map(|(k, &s)| {
|
||||||
|
let b = &html[s..starts.get(k + 1).copied().unwrap_or(html.len())];
|
||||||
let position = field(b, r#"(?s)<div id="denneuge[a-z]*"[^>]*>(.*?)</div>"#).parse().ok()?;
|
let position = field(b, r#"(?s)<div id="denneuge[a-z]*"[^>]*>(.*?)</div>"#).parse().ok()?;
|
||||||
let stat = |id: &str| field(b, &format!(r#"(?s)<div id="{id}"[^>]*>\s*<div id="listeikon">\d</div>(.*?)</div>"#));
|
let stat = |id: &str| field(b, &format!(r#"(?s)<div id="{id}"[^>]*>\s*<div id="listeikon">\d</div>(.*?)</div>"#));
|
||||||
Some(Row {
|
Some(Row {
|
||||||
@@ -58,44 +90,51 @@ pub fn parse(html: &str) -> Vec<Row> {
|
|||||||
last_week: stat("sidsteuge"),
|
last_week: stat("sidsteuge"),
|
||||||
weeks_on_chart: stat("ugerialt").parse().ok(),
|
weeks_on_chart: stat("ugerialt").parse().ok(),
|
||||||
peak: stat("peak").parse().ok(),
|
peak: stat("peak").parse().ok(),
|
||||||
artist: field(b, r#"(?s)<div id="artistnavnsmall">(.*?)</div>"#),
|
weeks_at_1: field(b, r#"(?s)<div id="ugersomnr1"[^>]*>(.*?)</div>"#).parse().ok(),
|
||||||
title: field(b, r#"(?s)<div id="titelbred">(.*?)</div>"#),
|
artist: field(b, r#"(?s)<div id="artistnavn(?:small)?">(.*?)</div>"#),
|
||||||
|
title: field(b, r#"(?s)<div id="titel(?:bred)?">(.*?)</div>"#),
|
||||||
label: field(b, r#"(?s)<div id="selskab">(.*?)</div>"#),
|
label: field(b, r#"(?s)<div id="selskab">(.*?)</div>"#),
|
||||||
|
spotify_id: spotify.captures(b).map(|c| c[1].to_string()),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.filter(|r| !r.artist.is_empty() || !r.title.is_empty())
|
.filter(|r| !r.artist.is_empty() || !r.title.is_empty())
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every week in the archive that isn't stored yet. `limit` caps the pages per run.
|
/// Every week of every chart that isn't stored yet. `limit` caps the pages per chart and run.
|
||||||
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||||
create_tables(db)?;
|
create_tables(db)?;
|
||||||
let first = http.get(&format!("{BASE}?list=airplay"))?.unwrap_or_default();
|
let mut fetched = 0;
|
||||||
let weeks: Vec<(i64, i64)> = Regex::new(r"w=(\d+)&y=(\d+)&list=airplay")?
|
for &(chart, size) in CHARTS {
|
||||||
|
let first = http.get(&format!("{BASE}?list={chart}"))?.unwrap_or_default();
|
||||||
|
let weeks: Vec<(i64, i64)> = Regex::new(&format!(r"w=(\d+)&y=(\d+)&list={chart}\b"))?
|
||||||
.captures_iter(&first)
|
.captures_iter(&first)
|
||||||
.filter_map(|c| Some((c[2].parse().ok()?, c[1].parse().ok()?)))
|
.filter_map(|c| Some((c[2].parse().ok()?, c[1].parse().ok()?)))
|
||||||
.collect();
|
.collect();
|
||||||
let mut stored = db.conn.prepare("SELECT count(*) FROM hitlisten_airplay WHERE year = ?1 AND week = ?2")?;
|
let mut stored = db.conn.prepare("SELECT count(*) FROM hitlisten_charts WHERE chart = ?1 AND year = ?2 AND week = ?3")?;
|
||||||
let mut fetched = 0;
|
let mut n = 0;
|
||||||
for (year, week) in weeks {
|
for (year, week) in weeks {
|
||||||
if fetched >= limit { break; }
|
if n >= limit { break; }
|
||||||
if stored.query_row(params![year, week], |r| r.get::<_, i64>(0))? > 0 { continue; }
|
if stored.query_row(params![chart, year, week], |r| r.get::<_, i64>(0))? > 0 { continue; }
|
||||||
let html = match http.get_bytes(&format!("{BASE}?w={week}&y={year}&list=airplay")) {
|
let html = match http.get_bytes(&format!("{BASE}?w={week}&y={year}&list={chart}")) {
|
||||||
// The site is Latin-1.
|
// The site is Latin-1.
|
||||||
Ok(Some(b)) => b.iter().map(|&c| c as char).collect::<String>(),
|
Ok(Some(b)) => b.iter().map(|&c| c as char).collect::<String>(),
|
||||||
Ok(None) => continue,
|
Ok(None) => continue,
|
||||||
Err(e) => { log.push(format!("hitlisten {year}-{week}: {e}")); continue; }
|
Err(e) => { log.push(format!("hitlisten {chart} {year}-{week}: {e}")); continue; }
|
||||||
};
|
};
|
||||||
let rows = parse(&html);
|
let rows = parse(&html);
|
||||||
if rows.is_empty() { log.push(format!("hitlisten {year}-{week}: no rows")); continue; }
|
if rows.is_empty() { log.push(format!("hitlisten {chart} {year}-{week}: no rows")); continue; }
|
||||||
if rows.len() != 20 { log.push(format!("hitlisten {year}-{week}: {} rows, not 20", rows.len())); }
|
if rows.len() != size { log.push(format!("hitlisten {chart} {year}-{week}: {} rows, not {size}", rows.len())); }
|
||||||
let tx = db.conn.unchecked_transaction()?;
|
let tx = db.conn.unchecked_transaction()?;
|
||||||
for r in rows {
|
for r in rows {
|
||||||
tx.execute("INSERT OR REPLACE INTO hitlisten_airplay VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
tx.execute("INSERT OR REPLACE INTO hitlisten_charts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
|
||||||
params![year, week, r.position, r.last_week, r.weeks_on_chart, r.peak, r.artist, r.title, r.label])?;
|
params![chart, year, week, r.position, r.last_week, r.weeks_on_chart, r.peak, r.weeks_at_1,
|
||||||
|
r.artist, r.title, r.label, r.spotify_id])?;
|
||||||
}
|
}
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
fetched += 1;
|
n += 1;
|
||||||
|
}
|
||||||
|
fetched += n;
|
||||||
}
|
}
|
||||||
Ok(fetched)
|
Ok(fetched)
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-5
@@ -110,8 +110,8 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
|||||||
step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
|
step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
|
||||||
}
|
}
|
||||||
if wants("hitlisten") {
|
if wants("hitlisten") {
|
||||||
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
|
let r = hitlisten::collect(db, &http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 60 }), &mut log);
|
||||||
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
|
step("hitlisten charts (airplay, track, album)", r.map(|n| format!("{n} weeks fetched")), &mut log);
|
||||||
}
|
}
|
||||||
db.conn.execute(
|
db.conn.execute(
|
||||||
"UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5",
|
"UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5",
|
||||||
@@ -138,10 +138,12 @@ fn stats(db: &db::Db) -> Result<()> {
|
|||||||
let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?;
|
let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?;
|
||||||
println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}");
|
println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}");
|
||||||
hitlisten::create_tables(db)?;
|
hitlisten::create_tables(db)?;
|
||||||
|
for &(chart, _) in hitlisten::CHARTS {
|
||||||
let (n, first, last): (i64, Option<String>, Option<String>) = db.conn.query_row(
|
let (n, first, last): (i64, Option<String>, Option<String>) = db.conn.query_row(
|
||||||
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay",
|
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_charts WHERE chart = ?1",
|
||||||
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
|
[chart], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
|
||||||
println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
|
println!("Hitlisten {chart}: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
|
||||||
|
}
|
||||||
deezer::create_tables(db)?;
|
deezer::create_tables(db)?;
|
||||||
let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row(
|
let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row(
|
||||||
"SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL),
|
"SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL),
|
||||||
|
|||||||
Reference in New Issue
Block a user