Song metadata: Deezer (ISRC, release date, label, type, genres, contributors) and MusicBrainz original release dates; data audit (airplay check)

Raw Deezer answers are kept so matches can be redone during cleaning. Every strong match's
release is stored, so the earliest release date can be taken (Deezer often returns reissues).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 01:14:11 +00:00
co-authored by Claude Opus 5.5
parent 6e8b258dcf
commit ccae36e11a
5 changed files with 376 additions and 2 deletions
+165
View File
@@ -0,0 +1,165 @@
//! Song metadata from Deezer's public API (no key): ISRC, release date, label, release type,
//! genres, contributors (featured artists) and popularity, for every distinct artist + title in
//! the plays and the charts.
//!
//! GET https://api.deezer.com/search?q={artist} {title} up to 25 candidate tracks
//! GET https://api.deezer.com/track/{id} isrc, release_date, contributors[], rank
//! GET https://api.deezer.com/album/{id} label, upc, record_type, genres, release_date
//!
//! Matching is deliberately simple here: the raw search answers are kept (deezer_searches.raw),
//! so the cleaning step can pick matches again without fetching. Rate limit: 50 requests / 5 s.
use crate::{db::Db, http::Http};
use anyhow::Result;
use chrono::Utc;
use rusqlite::{params, OptionalExtension};
use serde_json::Value;
const BASE: &str = "https://api.deezer.com";
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS deezer_searches (
artist TEXT NOT NULL, title TEXT NOT NULL, -- exactly as the station or chart wrote them
query TEXT NOT NULL, results INTEGER NOT NULL,
best_track_id INTEGER, best_score REAL, -- the simple matcher's pick (redo in cleaning)
fetched_at TEXT NOT NULL, raw TEXT NOT NULL,
PRIMARY KEY (artist, title)
);
CREATE TABLE IF NOT EXISTS deezer_tracks (
track_id INTEGER PRIMARY KEY, isrc TEXT, title TEXT, artist TEXT,
contributors TEXT, -- JSON [{name, role}]
release_date TEXT, duration_s INTEGER, rank INTEGER, explicit INTEGER,
album_id INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS deezer_tracks_isrc ON deezer_tracks (isrc);
CREATE TABLE IF NOT EXISTS deezer_albums (
album_id INTEGER PRIMARY KEY, title TEXT, upc TEXT, label TEXT, release_date TEXT,
record_type TEXT, genres TEXT, nb_tracks INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
);",
)?;
Ok(())
}
/// Lower-case letters and digits only, for comparing names.
fn key(s: &str) -> String {
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
}
/// The first artist ("Shakira & Burna Boy", "X ft. Y", "A, B" -> the first name) and the title
/// without version tags, for the search query only; the stored names stay as written.
fn query_parts(artist: &str, title: &str) -> (String, String) {
let lower = artist.to_lowercase();
let mut cut = artist.len();
for sep in [" feat.", " feat ", " ft.", " ft ", " featuring ", " & ", ", ", " x ", " og ", " [+] ", " med "] {
if let Some(i) = lower.find(sep) { cut = cut.min(i); }
}
let first = artist[..cut].trim().to_string();
let title = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title).trim().to_string();
(first, title)
}
/// 0..1: how well a candidate matches (title equal or contained, artist contained).
fn score(candidate: &Value, artist: &str, title: &str) -> f64 {
let (ct, ca) = (key(candidate["title"].as_str().unwrap_or_default()), key(candidate["artist"]["name"].as_str().unwrap_or_default()));
let (qt, qa) = (key(title), key(artist));
let t = if ct == qt { 1.0 } else if !qt.is_empty() && (ct.contains(&qt) || qt.contains(&ct)) { 0.7 } else { 0.0 };
let a = if ca == qa { 1.0 } else if !ca.is_empty() && (qa.contains(&ca) || ca.contains(&qa)) { 0.8 } else { 0.0 };
0.6 * t + 0.4 * a
}
fn fetch_json(http: &Http, url: &str) -> Result<Option<Value>> {
Ok(match http.get(url)? {
Some(b) => {
let v: Value = serde_json::from_str(&b)?;
if v.get("error").is_some() { anyhow::bail!("deezer: {}", v["error"]); }
Some(v)
}
None => None,
})
}
/// Looks up every artist + title not searched yet (`limit` per run).
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
crate::hitlisten::create_tables(db)?;
let todo: Vec<(String, String)> = db.conn.prepare(
"SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_airplay) s
WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title)
AND artist != '' AND title != ''")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
let now = Utc::now().to_rfc3339();
let mut done = 0;
for (artist, title) in todo.into_iter().take(limit) {
let (qa, qt) = query_parts(&artist, &title);
let query = format!("{qa} {qt}");
let url = format!("{BASE}/search?q={}&limit=10", urlencode(&query));
let found = match fetch_json(http, &url) {
Ok(v) => v.unwrap_or(Value::Null),
Err(e) => { log.push(format!("deezer search '{query}': {e}")); continue; }
};
let hits = found["data"].as_array().cloned().unwrap_or_default();
let best = hits.iter().map(|h| (score(h, &qa, &qt), h)).max_by(|a, b| a.0.total_cmp(&b.0));
let (best_id, best_score) = match best {
Some((s, h)) if s >= 0.6 => (h["id"].as_i64(), Some(s)),
Some((s, _)) => (None, Some(s)),
None => (None, None),
};
if let Some(id) = best_id {
if let Err(e) = track(db, http, id, &now) { log.push(format!("deezer track {id}: {e}")); }
}
// Every other strong match is another release of the same song (album, compilation,
// reissue): store those releases too, so cleaning can take the earliest release date.
for h in hits.iter().filter(|h| score(h, &qa, &qt) >= 0.9).take(6) {
if let Some(album_id) = h["album"]["id"].as_i64() {
if let Err(e) = album(db, http, album_id, &now) { log.push(format!("deezer album {album_id}: {e}")); }
}
}
db.conn.execute("INSERT OR REPLACE INTO deezer_searches VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![artist, title, query, hits.len() as i64, best_id, best_score, now, found.to_string()])?;
done += 1;
}
Ok(done)
}
/// Stores a track and its album (each fetched once).
fn track(db: &Db, http: &Http, id: i64, now: &str) -> Result<()> {
let have: Option<i64> = db.conn.query_row("SELECT album_id FROM deezer_tracks WHERE track_id = ?1", [id], |r| r.get(0)).optional()?;
let album_id = match have {
Some(a) => a,
None => {
let Some(t) = fetch_json(http, &format!("{BASE}/track/{id}"))? else { return Ok(()) };
let contributors: Vec<Value> = t["contributors"].as_array().cloned().unwrap_or_default().iter()
.map(|c| serde_json::json!({"name": c["name"], "role": c["role"], "id": c["id"]})).collect();
db.conn.execute("INSERT OR REPLACE INTO deezer_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![id, t["isrc"].as_str(), t["title"].as_str(), t["artist"]["name"].as_str(),
Value::Array(contributors).to_string(), t["release_date"].as_str(), t["duration"].as_i64(),
t["rank"].as_i64(), t["explicit_lyrics"].as_bool().map(|b| b as i64),
t["album"]["id"].as_i64(), now, t.to_string()])?;
t["album"]["id"].as_i64().unwrap_or(0)
}
};
if album_id != 0 { album(db, http, album_id, now)?; }
Ok(())
}
/// Stores a release once.
fn album(db: &Db, http: &Http, album_id: i64, now: &str) -> Result<()> {
let known: i64 = db.conn.query_row("SELECT count(*) FROM deezer_albums WHERE album_id = ?1", [album_id], |r| r.get(0))?;
if known > 0 { return Ok(()); }
if let Some(a) = fetch_json(http, &format!("{BASE}/album/{album_id}"))? {
let genres: Vec<&str> = a["genres"]["data"].as_array().map(|g| g.iter().filter_map(|x| x["name"].as_str()).collect()).unwrap_or_default();
db.conn.execute("INSERT OR REPLACE INTO deezer_albums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![album_id, a["title"].as_str(), a["upc"].as_str(), a["label"].as_str(), a["release_date"].as_str(),
a["record_type"].as_str(), genres.join(", "), a["nb_tracks"].as_i64(), now, a.to_string()])?;
}
Ok(())
}
fn urlencode(s: &str) -> String {
s.bytes().map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
b' ' => "%20".into(),
_ => format!("%{b:02X}"),
}).collect()
}