diff --git a/src/check.rs b/src/check.rs index 6144cab..37c8be6 100644 --- a/src/check.rs +++ b/src/check.rs @@ -5,7 +5,7 @@ //! DJ segues; checked by hand on P3: different songs, not duplicates) //! - DR cross-check: plays we counted per artist, channel and week against the weekly totals on //! DR's artist pages (two independent DR sources), for weeks we cover completely -//! - Hitlisten: weeks missing, and weeks without exactly 20 rows +//! - Hitlisten (airplay, Track and Album Top-40): weeks missing, weeks without a full chart use crate::{db::Db, hitlisten}; use anyhow::Result; @@ -102,26 +102,29 @@ fn dr_cross_check(db: &Db) -> Result<()> { } fn hitlisten_check(db: &Db) -> Result<()> { - println!("\n== Hitlisten Airplay Top-20"); hitlisten::create_tables(db)?; - let weeks: Vec<(i64, i64, i64)> = db.conn.prepare("SELECT year, week, count(*) FROM hitlisten_airplay GROUP BY year, week ORDER BY year, week")? - .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::>()?; - if weeks.is_empty() { println!("nothing collected"); return Ok(()); } - let odd: Vec = weeks.iter().filter(|w| w.2 != 20).map(|w| format!("{}-W{:02} ({} rows)", w.0, w.1, w.2)).collect(); - // Missing weeks between the first and the last (ISO weeks; some years have 53). - let have: std::collections::HashSet<(i64, i64)> = weeks.iter().map(|w| (w.0, w.1)).collect(); - let mut missing = Vec::new(); - let (fy, fw, ly, lw) = (weeks[0].0, weeks[0].1, weeks.last().unwrap().0, weeks.last().unwrap().1); - for y in fy..=ly { - let last_week = NaiveDate::from_ymd_opt(y as i32, 12, 28).map(|d| d.iso_week().week() as i64).unwrap_or(52); - for w in 1..=last_week { - if (y == fy && w < fw) || (y == ly && w > lw) { continue; } - if !have.contains(&(y, w)) { missing.push(format!("{y}-W{w:02}")); } + for &(chart, size) in hitlisten::CHARTS { + println!("\n== Hitlisten {chart}"); + let weeks: Vec<(i64, i64, i64)> = db.conn.prepare( + "SELECT year, week, count(*) FROM hitlisten_charts WHERE chart = ?1 GROUP BY year, week ORDER BY year, week")? + .query_map([chart], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::>()?; + if weeks.is_empty() { println!("nothing collected"); continue; } + let odd: Vec = weeks.iter().filter(|w| w.2 as usize != size).map(|w| format!("{}-W{:02} ({} rows)", w.0, w.1, w.2)).collect(); + // Missing weeks between the first and the last (ISO weeks; some years have 53). + let have: std::collections::HashSet<(i64, i64)> = weeks.iter().map(|w| (w.0, w.1)).collect(); + let mut missing = Vec::new(); + let (fy, fw, ly, lw) = (weeks[0].0, weeks[0].1, weeks.last().unwrap().0, weeks.last().unwrap().1); + for y in fy..=ly { + let last_week = NaiveDate::from_ymd_opt(y as i32, 12, 28).map(|d| d.iso_week().week() as i64).unwrap_or(52); + for w in 1..=last_week { + if (y == fy && w < fw) || (y == ly && w > lw) { continue; } + if !have.contains(&(y, w)) { missing.push(format!("{y}-W{w:02}")); } + } } + println!("{} weeks from {fy}-W{fw:02} to {ly}-W{lw:02}; {} missing; {} without {size} rows", + weeks.len(), missing.len(), odd.len()); + if !missing.is_empty() { println!("missing: {}", missing.iter().take(40).cloned().collect::>().join(", ")); } + if !odd.is_empty() { println!("odd: {}", odd.iter().take(20).cloned().collect::>().join(", ")); } } - println!("{} weeks from {fy}-W{fw:02} to {ly}-W{lw:02}; {} missing; {} without 20 rows", - weeks.len(), missing.len(), odd.len()); - if !missing.is_empty() { println!("missing: {}", missing.iter().take(40).cloned().collect::>().join(", ")); } - if !odd.is_empty() { println!("odd: {}", odd.iter().take(20).cloned().collect::>().join(", ")); } Ok(()) } diff --git a/src/deezer.rs b/src/deezer.rs index 5af17b6..5f08e72 100644 --- a/src/deezer.rs +++ b/src/deezer.rs @@ -84,7 +84,7 @@ pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Res create_tables(db)?; crate::hitlisten::create_tables(db)?; let todo: Vec<(String, String)> = db.conn.prepare( - "SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_airplay) s + "SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_charts WHERE chart != 'a40') s WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title) AND artist != '' AND title != ''")? .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; diff --git a/src/hitlisten.rs b/src/hitlisten.rs index d45bc4b..e7d896a 100644 --- a/src/hitlisten.rs +++ b/src/hitlisten.rs @@ -1,7 +1,11 @@ -//! Hitlisten (IFPI Danmark's official charts): the weekly Airplay Top-20, the most-played songs -//! on the biggest Danish radio stations (monitored by M&I Service), every week since week 38 of 2001. +//! Hitlisten (IFPI Danmark's official charts), every week of the archive: //! -//! https://www.hitlisten.nu/default.asp?w={week}&y={year}&list=airplay +//! airplay Airplay Top-20 since 2001-W38 the most-played songs on the biggest Danish radio +//! stations (monitored by M&I Service) +//! t40 Track Top-40 since 2007-W01 sales + streaming (streaming counted since Nov 2014) +//! a40 Album Top-40 since 2001 +//! +//! https://www.hitlisten.nu/default.asp?w={week}&y={year}&list={chart} //! //! The page's week selector lists every week that exists; weeks already stored aren't fetched again. @@ -12,16 +16,38 @@ use rusqlite::params; const BASE: &str = "https://www.hitlisten.nu/default.asp"; +/// (chart code, rows per week) +pub const CHARTS: &[(&str, usize)] = &[("airplay", 20), ("t40", 40), ("a40", 40)]; + pub fn create_tables(db: &Db) -> Result<()> { db.conn.execute_batch( - "CREATE TABLE IF NOT EXISTS hitlisten_airplay ( + "CREATE TABLE IF NOT EXISTS hitlisten_charts ( + chart TEXT NOT NULL, -- airplay | t40 | a40 year INTEGER NOT NULL, week INTEGER NOT NULL, position INTEGER NOT NULL, last_week TEXT, -- last week's position, 'NY' (new), 'RE' (re-entry) or empty - weeks_on_chart INTEGER, peak INTEGER, + weeks_on_chart INTEGER, peak INTEGER, weeks_at_1 INTEGER, artist TEXT NOT NULL, title TEXT NOT NULL, label TEXT, - PRIMARY KEY (year, week, position) + spotify_id TEXT, -- t40/a40: the release's Spotify track or album id, when linked + PRIMARY KEY (chart, year, week, position) );", )?; + // The first version stored the airplay chart on its own; move it in, keep the old name as a view. + let old_table: i64 = db.conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE name = 'hitlisten_airplay' AND type = 'table'", [], |r| r.get(0))?; + if old_table > 0 { + db.conn.execute_batch( + "BEGIN; + INSERT OR IGNORE INTO hitlisten_charts (chart, year, week, position, last_week, weeks_on_chart, peak, artist, title, label) + SELECT 'airplay', year, week, position, last_week, weeks_on_chart, peak, artist, title, label FROM hitlisten_airplay; + DROP TABLE hitlisten_airplay; + COMMIT;", + )?; + } + db.conn.execute_batch( + "CREATE VIEW IF NOT EXISTS hitlisten_airplay AS + SELECT year, week, position, last_week, weeks_on_chart, peak, artist, title, label + FROM hitlisten_charts WHERE chart = 'airplay';", + )?; Ok(()) } @@ -32,25 +58,31 @@ fn text(s: &str) -> String { pub struct Row { pub position: i64, - pub last_week: String, // last week's position, "NY" (new), "RE" (re-entry) or empty + pub last_week: String, pub weeks_on_chart: Option, pub peak: Option, + pub weeks_at_1: Option, pub artist: String, pub title: String, pub label: String, + pub spotify_id: Option, } -/// The chart rows of one week's page. Each row is a
block; the fields -/// are read one by one so markup variants (re-entries, bold NY/RE) don't drop rows. +/// The chart rows of one week's page. The airplay chart's rows are
+/// blocks, the Top-40s' are
; the fields are read one by one so markup +/// variants (re-entries, bold NY/RE, links around the label) don't drop rows. pub fn parse(html: &str) -> Vec { - let tags = Regex::new(r"<[^>]+>").unwrap(); + let tags = Regex::new(r"(?s)|<[^>]+>").unwrap(); let field = |block: &str, re: &str| -> String { Regex::new(re).unwrap().captures(block) .map(|c| text(&tags.replace_all(&c[1], " "))).unwrap_or_default() }; - html.split(r#"
"#) - .skip(1) - .filter_map(|b| { + let spotify = Regex::new(r"open\.spotify\.com/(?:track|album)/([A-Za-z0-9]{22})").unwrap(); + let block_start = Regex::new(r#"
|
"#).unwrap(); + let starts: Vec = block_start.find_iter(html).map(|m| m.start()).collect(); + starts.iter().enumerate() + .filter_map(|(k, &s)| { + let b = &html[s..starts.get(k + 1).copied().unwrap_or(html.len())]; let position = field(b, r#"(?s)
]*>(.*?)
"#).parse().ok()?; let stat = |id: &str| field(b, &format!(r#"(?s)
]*>\s*
\d
(.*?)
"#)); Some(Row { @@ -58,44 +90,51 @@ pub fn parse(html: &str) -> Vec { last_week: stat("sidsteuge"), weeks_on_chart: stat("ugerialt").parse().ok(), peak: stat("peak").parse().ok(), - artist: field(b, r#"(?s)
(.*?)
"#), - title: field(b, r#"(?s)
(.*?)
"#), + weeks_at_1: field(b, r#"(?s)
]*>(.*?)
"#).parse().ok(), + artist: field(b, r#"(?s)
(.*?)
"#), + title: field(b, r#"(?s)
(.*?)
"#), label: field(b, r#"(?s)
(.*?)
"#), + spotify_id: spotify.captures(b).map(|c| c[1].to_string()), }) }) .filter(|r| !r.artist.is_empty() || !r.title.is_empty()) .collect() } -/// Every week in the archive that isn't stored yet. `limit` caps the pages per run. +/// Every week of every chart that isn't stored yet. `limit` caps the pages per chart and run. pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { create_tables(db)?; - let first = http.get(&format!("{BASE}?list=airplay"))?.unwrap_or_default(); - let weeks: Vec<(i64, i64)> = Regex::new(r"w=(\d+)&y=(\d+)&list=airplay")? - .captures_iter(&first) - .filter_map(|c| Some((c[2].parse().ok()?, c[1].parse().ok()?))) - .collect(); - let mut stored = db.conn.prepare("SELECT count(*) FROM hitlisten_airplay WHERE year = ?1 AND week = ?2")?; let mut fetched = 0; - for (year, week) in weeks { - if fetched >= limit { break; } - if stored.query_row(params![year, week], |r| r.get::<_, i64>(0))? > 0 { continue; } - let html = match http.get_bytes(&format!("{BASE}?w={week}&y={year}&list=airplay")) { - // The site is Latin-1. - Ok(Some(b)) => b.iter().map(|&c| c as char).collect::(), - Ok(None) => continue, - Err(e) => { log.push(format!("hitlisten {year}-{week}: {e}")); continue; } - }; - let rows = parse(&html); - if rows.is_empty() { log.push(format!("hitlisten {year}-{week}: no rows")); continue; } - if rows.len() != 20 { log.push(format!("hitlisten {year}-{week}: {} rows, not 20", rows.len())); } - let tx = db.conn.unchecked_transaction()?; - for r in rows { - tx.execute("INSERT OR REPLACE INTO hitlisten_airplay VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", - params![year, week, r.position, r.last_week, r.weeks_on_chart, r.peak, r.artist, r.title, r.label])?; + for &(chart, size) in CHARTS { + let first = http.get(&format!("{BASE}?list={chart}"))?.unwrap_or_default(); + let weeks: Vec<(i64, i64)> = Regex::new(&format!(r"w=(\d+)&y=(\d+)&list={chart}\b"))? + .captures_iter(&first) + .filter_map(|c| Some((c[2].parse().ok()?, c[1].parse().ok()?))) + .collect(); + let mut stored = db.conn.prepare("SELECT count(*) FROM hitlisten_charts WHERE chart = ?1 AND year = ?2 AND week = ?3")?; + let mut n = 0; + for (year, week) in weeks { + if n >= limit { break; } + if stored.query_row(params![chart, year, week], |r| r.get::<_, i64>(0))? > 0 { continue; } + let html = match http.get_bytes(&format!("{BASE}?w={week}&y={year}&list={chart}")) { + // The site is Latin-1. + Ok(Some(b)) => b.iter().map(|&c| c as char).collect::(), + Ok(None) => continue, + Err(e) => { log.push(format!("hitlisten {chart} {year}-{week}: {e}")); continue; } + }; + let rows = parse(&html); + if rows.is_empty() { log.push(format!("hitlisten {chart} {year}-{week}: no rows")); continue; } + if rows.len() != size { log.push(format!("hitlisten {chart} {year}-{week}: {} rows, not {size}", rows.len())); } + let tx = db.conn.unchecked_transaction()?; + for r in rows { + tx.execute("INSERT OR REPLACE INTO hitlisten_charts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![chart, year, week, r.position, r.last_week, r.weeks_on_chart, r.peak, r.weeks_at_1, + r.artist, r.title, r.label, r.spotify_id])?; + } + tx.commit()?; + n += 1; } - tx.commit()?; - fetched += 1; + fetched += n; } Ok(fetched) } diff --git a/src/main.rs b/src/main.rs index 1eb04e0..e1a21d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -110,8 +110,8 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log); } if wants("hitlisten") { - let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log); - step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log); + let r = hitlisten::collect(db, &http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 60 }), &mut log); + step("hitlisten charts (airplay, track, album)", r.map(|n| format!("{n} weeks fetched")), &mut log); } db.conn.execute( "UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5", @@ -138,10 +138,12 @@ fn stats(db: &db::Db) -> Result<()> { let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?; println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}"); hitlisten::create_tables(db)?; - let (n, first, last): (i64, Option, Option) = db.conn.query_row( - "SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay", - [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; - println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default()); + for &(chart, _) in hitlisten::CHARTS { + let (n, first, last): (i64, Option, Option) = db.conn.query_row( + "SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_charts WHERE chart = ?1", + [chart], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; + println!("Hitlisten {chart}: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default()); + } deezer::create_tables(db)?; let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row( "SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL),