227 lines
12 KiB
Rust
227 lines
12 KiB
Rust
//! Song metadata from Deezer's public API (no key): ISRC, release date, label, release type,
|
|
//! genres, contributors (featured artists) and popularity, for every distinct artist + title in
|
|
//! the plays and the charts.
|
|
//!
|
|
//! GET https://api.deezer.com/search?q={artist} {title} up to 25 candidate tracks
|
|
//! GET https://api.deezer.com/track/{id} isrc, release_date, contributors[], rank
|
|
//! GET https://api.deezer.com/album/{id} label, upc, record_type, genres, release_date
|
|
//!
|
|
//! Matching is deliberately simple here: the raw search answers are kept (deezer_searches.raw),
|
|
//! so the cleaning step can pick matches again without fetching. Rate limit: 50 requests / 5 s.
|
|
|
|
use crate::{db::Db, http::Http};
|
|
use anyhow::Result;
|
|
use chrono::Utc;
|
|
use rusqlite::{params, OptionalExtension};
|
|
use serde_json::Value;
|
|
|
|
const BASE: &str = "https://api.deezer.com";
|
|
|
|
pub fn create_tables(db: &Db) -> Result<()> {
|
|
db.conn.execute_batch(
|
|
"CREATE TABLE IF NOT EXISTS deezer_searches (
|
|
artist TEXT NOT NULL, title TEXT NOT NULL, -- exactly as the station or chart wrote them
|
|
query TEXT NOT NULL, results INTEGER NOT NULL,
|
|
best_track_id INTEGER, best_score REAL, -- the simple matcher's pick (redo in cleaning)
|
|
fetched_at TEXT NOT NULL, raw TEXT NOT NULL,
|
|
PRIMARY KEY (artist, title)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS deezer_tracks (
|
|
track_id INTEGER PRIMARY KEY, isrc TEXT, title TEXT, artist TEXT,
|
|
contributors TEXT, -- JSON [{name, role}]
|
|
release_date TEXT, duration_s INTEGER, rank INTEGER, explicit INTEGER,
|
|
album_id INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS deezer_tracks_isrc ON deezer_tracks (isrc);
|
|
CREATE TABLE IF NOT EXISTS deezer_albums (
|
|
album_id INTEGER PRIMARY KEY, title TEXT, upc TEXT, label TEXT, release_date TEXT,
|
|
record_type TEXT, genres TEXT, nb_tracks INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
|
|
);",
|
|
)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Lower-case letters and digits only, for comparing names.
|
|
fn key(s: &str) -> String {
|
|
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
|
|
}
|
|
|
|
/// The first artist ("Shakira & Burna Boy", "X ft. Y", "A, B" -> the first name) and the title
|
|
/// without version tags, for the search query only; the stored names stay as written.
|
|
fn query_parts(artist: &str, title: &str) -> (String, String) {
|
|
let lower = artist.to_lowercase();
|
|
let mut cut = artist.len();
|
|
for sep in [" feat.", " feat ", " ft.", " ft ", " featuring ", " & ", ", ", " x ", " og ", " [+] ", " med "] {
|
|
if let Some(i) = lower.find(sep) { cut = cut.min(i); }
|
|
}
|
|
let first = artist[..cut].trim().to_string();
|
|
let title = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title).trim().to_string();
|
|
(first, title)
|
|
}
|
|
|
|
/// 0..1: how well a candidate matches (title equal or contained, artist contained).
|
|
fn score(candidate: &Value, artist: &str, title: &str) -> f64 {
|
|
let (ct, ca) = (key(candidate["title"].as_str().unwrap_or_default()), key(candidate["artist"]["name"].as_str().unwrap_or_default()));
|
|
let (qt, qa) = (key(title), key(artist));
|
|
let t = if ct == qt { 1.0 } else if !qt.is_empty() && (ct.contains(&qt) || qt.contains(&ct)) { 0.7 } else { 0.0 };
|
|
let a = if ca == qa { 1.0 } else if !ca.is_empty() && (qa.contains(&ca) || ca.contains(&qa)) { 0.8 } else { 0.0 };
|
|
0.6 * t + 0.4 * a
|
|
}
|
|
|
|
fn fetch_json(http: &Http, url: &str) -> Result<Option<Value>> {
|
|
Ok(match http.get(url)? {
|
|
Some(b) => {
|
|
let v: Value = serde_json::from_str(&b)?;
|
|
if v.get("error").is_some() { anyhow::bail!("deezer: {}", v["error"]); }
|
|
Some(v)
|
|
}
|
|
None => None,
|
|
})
|
|
}
|
|
|
|
/// Looks up every artist + title not searched yet (`limit` per run).
|
|
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
|
create_tables(db)?;
|
|
crate::hitlisten::create_tables(db)?;
|
|
let todo: Vec<(String, String)> = db.conn.prepare(
|
|
"SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_charts WHERE chart != 'a40') s
|
|
WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title)
|
|
AND artist != '' AND title != ''")?
|
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
|
let now = Utc::now().to_rfc3339();
|
|
let mut done = 0;
|
|
for (artist, title) in todo.into_iter().take(limit) {
|
|
let (qa, qt) = query_parts(&artist, &title);
|
|
let query = format!("{qa} {qt}");
|
|
let url = format!("{BASE}/search?q={}&limit=10", urlencode(&query));
|
|
let found = match fetch_json(http, &url) {
|
|
Ok(v) => v.unwrap_or(Value::Null),
|
|
Err(e) => { log.push(format!("deezer search '{query}': {e}")); continue; }
|
|
};
|
|
let hits = found["data"].as_array().cloned().unwrap_or_default();
|
|
let best = hits.iter().map(|h| (score(h, &qa, &qt), h)).max_by(|a, b| a.0.total_cmp(&b.0));
|
|
let (best_id, best_score) = match best {
|
|
Some((s, h)) if s >= 0.6 => (h["id"].as_i64(), Some(s)),
|
|
Some((s, _)) => (None, Some(s)),
|
|
None => (None, None),
|
|
};
|
|
if let Some(id) = best_id {
|
|
if let Err(e) = track(db, http, id, &now) { log.push(format!("deezer track {id}: {e}")); }
|
|
}
|
|
// Every other strong match is another release of the same song (album, compilation,
|
|
// reissue): store those releases too, so cleaning can take the earliest release date.
|
|
for h in hits.iter().filter(|h| score(h, &qa, &qt) >= 0.9).take(6) {
|
|
if let Some(album_id) = h["album"]["id"].as_i64() {
|
|
if let Err(e) = album(db, http, album_id, &now) { log.push(format!("deezer album {album_id}: {e}")); }
|
|
}
|
|
}
|
|
db.conn.execute("INSERT OR REPLACE INTO deezer_searches VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
|
params![artist, title, query, hits.len() as i64, best_id, best_score, now, found.to_string()])?;
|
|
done += 1;
|
|
}
|
|
Ok(done)
|
|
}
|
|
|
|
/// Stores a track and its album (each fetched once).
|
|
fn track(db: &Db, http: &Http, id: i64, now: &str) -> Result<()> {
|
|
let have: Option<i64> = db.conn.query_row("SELECT album_id FROM deezer_tracks WHERE track_id = ?1", [id], |r| r.get(0)).optional()?;
|
|
let album_id = match have {
|
|
Some(a) => a,
|
|
None => {
|
|
let Some(t) = fetch_json(http, &format!("{BASE}/track/{id}"))? else { return Ok(()) };
|
|
let contributors: Vec<Value> = t["contributors"].as_array().cloned().unwrap_or_default().iter()
|
|
.map(|c| serde_json::json!({"name": c["name"], "role": c["role"], "id": c["id"]})).collect();
|
|
db.conn.execute("INSERT OR REPLACE INTO deezer_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
|
|
params![id, t["isrc"].as_str(), t["title"].as_str(), t["artist"]["name"].as_str(),
|
|
Value::Array(contributors).to_string(), t["release_date"].as_str(), t["duration"].as_i64(),
|
|
t["rank"].as_i64(), t["explicit_lyrics"].as_bool().map(|b| b as i64),
|
|
t["album"]["id"].as_i64(), now, t.to_string()])?;
|
|
t["album"]["id"].as_i64().unwrap_or(0)
|
|
}
|
|
};
|
|
if album_id != 0 { album(db, http, album_id, now)?; }
|
|
Ok(())
|
|
}
|
|
|
|
/// Stores a release once.
|
|
fn album(db: &Db, http: &Http, album_id: i64, now: &str) -> Result<()> {
|
|
let known: i64 = db.conn.query_row("SELECT count(*) FROM deezer_albums WHERE album_id = ?1", [album_id], |r| r.get(0))?;
|
|
if known > 0 { return Ok(()); }
|
|
if let Some(a) = fetch_json(http, &format!("{BASE}/album/{album_id}"))? {
|
|
let genres: Vec<&str> = a["genres"]["data"].as_array().map(|g| g.iter().filter_map(|x| x["name"].as_str()).collect()).unwrap_or_default();
|
|
db.conn.execute("INSERT OR REPLACE INTO deezer_albums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
|
params![album_id, a["title"].as_str(), a["upc"].as_str(), a["label"].as_str(), a["release_date"].as_str(),
|
|
a["record_type"].as_str(), genres.join(", "), a["nb_tracks"].as_i64(), now, a.to_string()])?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn urlencode(s: &str) -> String {
|
|
s.bytes().map(|b| match b {
|
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
|
|
b' ' => "%20".into(),
|
|
_ => format!("%{b:02X}"),
|
|
}).collect()
|
|
}
|
|
|
|
// ------------------------------------------------------------------ discographies
|
|
|
|
/// Every DR artist's releases from Deezer, for release dates and the historical candidate pool:
|
|
/// GET /search/artist?q={name} the Deezer artist: exact name match, most fans
|
|
/// GET /artist/{id}/albums every release (album, EP, single) with its date
|
|
pub fn create_discography_tables(db: &Db) -> Result<()> {
|
|
db.conn.execute_batch(
|
|
"CREATE TABLE IF NOT EXISTS deezer_artists (
|
|
dr_artist_id TEXT PRIMARY KEY, deezer_artist_id INTEGER, name TEXT, fans INTEGER,
|
|
candidates INTEGER, fetched_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS deezer_releases (
|
|
deezer_artist_id INTEGER NOT NULL, album_id INTEGER NOT NULL, title TEXT, release_date TEXT,
|
|
record_type TEXT, explicit INTEGER, fetched_at TEXT NOT NULL,
|
|
PRIMARY KEY (deezer_artist_id, album_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS deezer_releases_date ON deezer_releases (release_date);",
|
|
)?;
|
|
Ok(())
|
|
}
|
|
|
|
pub fn discographies(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
|
create_discography_tables(db)?;
|
|
let todo: Vec<(String, String)> = db.conn.prepare(
|
|
"SELECT artist_id, name FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM deezer_artists) ORDER BY artist_id")?
|
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
|
let now = Utc::now().to_rfc3339();
|
|
let mut done = 0;
|
|
for (dr_id, name) in todo.into_iter().take(limit) {
|
|
let found = match fetch_json(http, &format!("{BASE}/search/artist?q={}&limit=10", urlencode(&name))) {
|
|
Ok(v) => v.unwrap_or(Value::Null),
|
|
Err(e) => { log.push(format!("deezer artist '{name}': {e}")); continue; }
|
|
};
|
|
let hits = found["data"].as_array().cloned().unwrap_or_default();
|
|
let best = hits.iter().filter(|h| key(h["name"].as_str().unwrap_or_default()) == key(&name))
|
|
.max_by_key(|h| h["nb_fan"].as_i64().unwrap_or(0));
|
|
let (did, fans) = (best.and_then(|h| h["id"].as_i64()), best.and_then(|h| h["nb_fan"].as_i64()));
|
|
if let Some(id) = did {
|
|
let have: i64 = db.conn.query_row("SELECT count(*) FROM deezer_releases WHERE deezer_artist_id = ?1", [id], |r| r.get(0))?;
|
|
if have == 0 {
|
|
let mut url = Some(format!("{BASE}/artist/{id}/albums?limit=100"));
|
|
let mut pages = 0;
|
|
while let Some(u) = url.take() {
|
|
let v = match fetch_json(http, &u) { Ok(Some(v)) => v, Ok(None) => break, Err(e) => { log.push(format!("deezer albums {id}: {e}")); break; } };
|
|
for a in v["data"].as_array().into_iter().flatten() {
|
|
db.conn.execute("INSERT OR REPLACE INTO deezer_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
|
params![id, a["id"].as_i64(), a["title"].as_str(), a["release_date"].as_str(), a["record_type"].as_str(),
|
|
a["explicit_lyrics"].as_bool().map(|b| b as i64), now])?;
|
|
}
|
|
pages += 1;
|
|
url = v["next"].as_str().filter(|_| pages < 5).map(str::to_string);
|
|
}
|
|
}
|
|
}
|
|
db.conn.execute("INSERT OR REPLACE INTO deezer_artists VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
|
params![dr_id, did, name, fans, hits.len() as i64, now])?;
|
|
done += 1;
|
|
}
|
|
Ok(done)
|
|
}
|