//! Song metadata from Deezer's public API (no key): ISRC, release date, label, release type, //! genres, contributors (featured artists) and popularity, for every distinct artist + title in //! the plays and the charts. //! //! GET https://api.deezer.com/search?q={artist} {title} up to 25 candidate tracks //! GET https://api.deezer.com/track/{id} isrc, release_date, contributors[], rank //! GET https://api.deezer.com/album/{id} label, upc, record_type, genres, release_date //! //! Matching is deliberately simple here: the raw search answers are kept (deezer_searches.raw), //! so the cleaning step can pick matches again without fetching. Rate limit: 50 requests / 5 s. use crate::{db::Db, http::Http}; use anyhow::Result; use chrono::Utc; use rusqlite::{params, OptionalExtension}; use serde_json::Value; const BASE: &str = "https://api.deezer.com"; pub fn create_tables(db: &Db) -> Result<()> { db.conn.execute_batch( "CREATE TABLE IF NOT EXISTS deezer_searches ( artist TEXT NOT NULL, title TEXT NOT NULL, -- exactly as the station or chart wrote them query TEXT NOT NULL, results INTEGER NOT NULL, best_track_id INTEGER, best_score REAL, -- the simple matcher's pick (redo in cleaning) fetched_at TEXT NOT NULL, raw TEXT NOT NULL, PRIMARY KEY (artist, title) ); CREATE TABLE IF NOT EXISTS deezer_tracks ( track_id INTEGER PRIMARY KEY, isrc TEXT, title TEXT, artist TEXT, contributors TEXT, -- JSON [{name, role}] release_date TEXT, duration_s INTEGER, rank INTEGER, explicit INTEGER, album_id INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS deezer_tracks_isrc ON deezer_tracks (isrc); CREATE TABLE IF NOT EXISTS deezer_albums ( album_id INTEGER PRIMARY KEY, title TEXT, upc TEXT, label TEXT, release_date TEXT, record_type TEXT, genres TEXT, nb_tracks INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL );", )?; Ok(()) } /// Lower-case letters and digits only, for comparing names. fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() } /// The first artist ("Shakira & Burna Boy", "X ft. Y", "A, B" -> the first name) and the title /// without version tags, for the search query only; the stored names stay as written. fn query_parts(artist: &str, title: &str) -> (String, String) { let lower = artist.to_lowercase(); let mut cut = artist.len(); for sep in [" feat.", " feat ", " ft.", " ft ", " featuring ", " & ", ", ", " x ", " og ", " [+] ", " med "] { if let Some(i) = lower.find(sep) { cut = cut.min(i); } } let first = artist[..cut].trim().to_string(); let title = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title).trim().to_string(); (first, title) } /// 0..1: how well a candidate matches (title equal or contained, artist contained). fn score(candidate: &Value, artist: &str, title: &str) -> f64 { let (ct, ca) = (key(candidate["title"].as_str().unwrap_or_default()), key(candidate["artist"]["name"].as_str().unwrap_or_default())); let (qt, qa) = (key(title), key(artist)); let t = if ct == qt { 1.0 } else if !qt.is_empty() && (ct.contains(&qt) || qt.contains(&ct)) { 0.7 } else { 0.0 }; let a = if ca == qa { 1.0 } else if !ca.is_empty() && (qa.contains(&ca) || ca.contains(&qa)) { 0.8 } else { 0.0 }; 0.6 * t + 0.4 * a } fn fetch_json(http: &Http, url: &str) -> Result> { Ok(match http.get(url)? { Some(b) => { let v: Value = serde_json::from_str(&b)?; if v.get("error").is_some() { anyhow::bail!("deezer: {}", v["error"]); } Some(v) } None => None, }) } /// Looks up every artist + title not searched yet (`limit` per run). pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { create_tables(db)?; crate::hitlisten::create_tables(db)?; let todo: Vec<(String, String)> = db.conn.prepare( "SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_charts WHERE chart != 'a40') s WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title) AND artist != '' AND title != ''")? .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; let now = Utc::now().to_rfc3339(); let mut done = 0; for (artist, title) in todo.into_iter().take(limit) { let (qa, qt) = query_parts(&artist, &title); let query = format!("{qa} {qt}"); let url = format!("{BASE}/search?q={}&limit=10", urlencode(&query)); let found = match fetch_json(http, &url) { Ok(v) => v.unwrap_or(Value::Null), Err(e) => { log.push(format!("deezer search '{query}': {e}")); continue; } }; let hits = found["data"].as_array().cloned().unwrap_or_default(); let best = hits.iter().map(|h| (score(h, &qa, &qt), h)).max_by(|a, b| a.0.total_cmp(&b.0)); let (best_id, best_score) = match best { Some((s, h)) if s >= 0.6 => (h["id"].as_i64(), Some(s)), Some((s, _)) => (None, Some(s)), None => (None, None), }; if let Some(id) = best_id { if let Err(e) = track(db, http, id, &now) { log.push(format!("deezer track {id}: {e}")); } } // Every other strong match is another release of the same song (album, compilation, // reissue): store those releases too, so cleaning can take the earliest release date. for h in hits.iter().filter(|h| score(h, &qa, &qt) >= 0.9).take(6) { if let Some(album_id) = h["album"]["id"].as_i64() { if let Err(e) = album(db, http, album_id, &now) { log.push(format!("deezer album {album_id}: {e}")); } } } db.conn.execute("INSERT OR REPLACE INTO deezer_searches VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![artist, title, query, hits.len() as i64, best_id, best_score, now, found.to_string()])?; done += 1; } Ok(done) } /// Stores a track and its album (each fetched once). fn track(db: &Db, http: &Http, id: i64, now: &str) -> Result<()> { let have: Option = db.conn.query_row("SELECT album_id FROM deezer_tracks WHERE track_id = ?1", [id], |r| r.get(0)).optional()?; let album_id = match have { Some(a) => a, None => { let Some(t) = fetch_json(http, &format!("{BASE}/track/{id}"))? else { return Ok(()) }; let contributors: Vec = t["contributors"].as_array().cloned().unwrap_or_default().iter() .map(|c| serde_json::json!({"name": c["name"], "role": c["role"], "id": c["id"]})).collect(); db.conn.execute("INSERT OR REPLACE INTO deezer_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", params![id, t["isrc"].as_str(), t["title"].as_str(), t["artist"]["name"].as_str(), Value::Array(contributors).to_string(), t["release_date"].as_str(), t["duration"].as_i64(), t["rank"].as_i64(), t["explicit_lyrics"].as_bool().map(|b| b as i64), t["album"]["id"].as_i64(), now, t.to_string()])?; t["album"]["id"].as_i64().unwrap_or(0) } }; if album_id != 0 { album(db, http, album_id, now)?; } Ok(()) } /// Stores a release once. fn album(db: &Db, http: &Http, album_id: i64, now: &str) -> Result<()> { let known: i64 = db.conn.query_row("SELECT count(*) FROM deezer_albums WHERE album_id = ?1", [album_id], |r| r.get(0))?; if known > 0 { return Ok(()); } if let Some(a) = fetch_json(http, &format!("{BASE}/album/{album_id}"))? { let genres: Vec<&str> = a["genres"]["data"].as_array().map(|g| g.iter().filter_map(|x| x["name"].as_str()).collect()).unwrap_or_default(); db.conn.execute("INSERT OR REPLACE INTO deezer_albums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", params![album_id, a["title"].as_str(), a["upc"].as_str(), a["label"].as_str(), a["release_date"].as_str(), a["record_type"].as_str(), genres.join(", "), a["nb_tracks"].as_i64(), now, a.to_string()])?; } Ok(()) } fn urlencode(s: &str) -> String { s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), b' ' => "%20".into(), _ => format!("%{b:02X}"), }).collect() } // ------------------------------------------------------------------ discographies /// Every DR artist's releases from Deezer, for release dates and the historical candidate pool: /// GET /search/artist?q={name} the Deezer artist: exact name match, most fans /// GET /artist/{id}/albums every release (album, EP, single) with its date pub fn create_discography_tables(db: &Db) -> Result<()> { db.conn.execute_batch( "CREATE TABLE IF NOT EXISTS deezer_artists ( dr_artist_id TEXT PRIMARY KEY, deezer_artist_id INTEGER, name TEXT, fans INTEGER, candidates INTEGER, fetched_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS deezer_releases ( deezer_artist_id INTEGER NOT NULL, album_id INTEGER NOT NULL, title TEXT, release_date TEXT, record_type TEXT, explicit INTEGER, fetched_at TEXT NOT NULL, PRIMARY KEY (deezer_artist_id, album_id) ); CREATE INDEX IF NOT EXISTS deezer_releases_date ON deezer_releases (release_date);", )?; Ok(()) } pub fn discographies(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { create_discography_tables(db)?; let todo: Vec<(String, String)> = db.conn.prepare( "SELECT artist_id, name FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM deezer_artists) ORDER BY artist_id")? .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; let now = Utc::now().to_rfc3339(); let mut done = 0; for (dr_id, name) in todo.into_iter().take(limit) { let found = match fetch_json(http, &format!("{BASE}/search/artist?q={}&limit=10", urlencode(&name))) { Ok(v) => v.unwrap_or(Value::Null), Err(e) => { log.push(format!("deezer artist '{name}': {e}")); continue; } }; let hits = found["data"].as_array().cloned().unwrap_or_default(); let best = hits.iter().filter(|h| key(h["name"].as_str().unwrap_or_default()) == key(&name)) .max_by_key(|h| h["nb_fan"].as_i64().unwrap_or(0)); let (did, fans) = (best.and_then(|h| h["id"].as_i64()), best.and_then(|h| h["nb_fan"].as_i64())); if let Some(id) = did { let have: i64 = db.conn.query_row("SELECT count(*) FROM deezer_releases WHERE deezer_artist_id = ?1", [id], |r| r.get(0))?; if have == 0 { let mut url = Some(format!("{BASE}/artist/{id}/albums?limit=100")); let mut pages = 0; while let Some(u) = url.take() { let v = match fetch_json(http, &u) { Ok(Some(v)) => v, Ok(None) => break, Err(e) => { log.push(format!("deezer albums {id}: {e}")); break; } }; for a in v["data"].as_array().into_iter().flatten() { db.conn.execute("INSERT OR REPLACE INTO deezer_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", params![id, a["id"].as_i64(), a["title"].as_str(), a["release_date"].as_str(), a["record_type"].as_str(), a["explicit_lyrics"].as_bool().map(|b| b as i64), now])?; } pages += 1; url = v["next"].as_str().filter(|_| pages < 5).map(str::to_string); } } } db.conn.execute("INSERT OR REPLACE INTO deezer_artists VALUES (?1, ?2, ?3, ?4, ?5, ?6)", params![dr_id, did, name, fans, hits.len() as i64, now])?; done += 1; } Ok(done) }