Files
Airplay-Predictor/src/deezer.rs
T

227 lines
12 KiB
Rust

//! Song metadata from Deezer's public API (no key): ISRC, release date, label, release type,
//! genres, contributors (featured artists) and popularity, for every distinct artist + title in
//! the plays and the charts.
//!
//! GET https://api.deezer.com/search?q={artist} {title} up to 25 candidate tracks
//! GET https://api.deezer.com/track/{id} isrc, release_date, contributors[], rank
//! GET https://api.deezer.com/album/{id} label, upc, record_type, genres, release_date
//!
//! Matching is deliberately simple here: the raw search answers are kept (deezer_searches.raw),
//! so the cleaning step can pick matches again without fetching. Rate limit: 50 requests / 5 s.
use crate::{db::Db, http::Http};
use anyhow::Result;
use chrono::Utc;
use rusqlite::{params, OptionalExtension};
use serde_json::Value;
const BASE: &str = "https://api.deezer.com";
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS deezer_searches (
artist TEXT NOT NULL, title TEXT NOT NULL, -- exactly as the station or chart wrote them
query TEXT NOT NULL, results INTEGER NOT NULL,
best_track_id INTEGER, best_score REAL, -- the simple matcher's pick (redo in cleaning)
fetched_at TEXT NOT NULL, raw TEXT NOT NULL,
PRIMARY KEY (artist, title)
);
CREATE TABLE IF NOT EXISTS deezer_tracks (
track_id INTEGER PRIMARY KEY, isrc TEXT, title TEXT, artist TEXT,
contributors TEXT, -- JSON [{name, role}]
release_date TEXT, duration_s INTEGER, rank INTEGER, explicit INTEGER,
album_id INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS deezer_tracks_isrc ON deezer_tracks (isrc);
CREATE TABLE IF NOT EXISTS deezer_albums (
album_id INTEGER PRIMARY KEY, title TEXT, upc TEXT, label TEXT, release_date TEXT,
record_type TEXT, genres TEXT, nb_tracks INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
);",
)?;
Ok(())
}
/// Lower-case letters and digits only, for comparing names.
fn key(s: &str) -> String {
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
}
/// The first artist ("Shakira & Burna Boy", "X ft. Y", "A, B" -> the first name) and the title
/// without version tags, for the search query only; the stored names stay as written.
fn query_parts(artist: &str, title: &str) -> (String, String) {
let lower = artist.to_lowercase();
let mut cut = artist.len();
for sep in [" feat.", " feat ", " ft.", " ft ", " featuring ", " & ", ", ", " x ", " og ", " [+] ", " med "] {
if let Some(i) = lower.find(sep) { cut = cut.min(i); }
}
let first = artist[..cut].trim().to_string();
let title = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title).trim().to_string();
(first, title)
}
/// 0..1: how well a candidate matches (title equal or contained, artist contained).
fn score(candidate: &Value, artist: &str, title: &str) -> f64 {
let (ct, ca) = (key(candidate["title"].as_str().unwrap_or_default()), key(candidate["artist"]["name"].as_str().unwrap_or_default()));
let (qt, qa) = (key(title), key(artist));
let t = if ct == qt { 1.0 } else if !qt.is_empty() && (ct.contains(&qt) || qt.contains(&ct)) { 0.7 } else { 0.0 };
let a = if ca == qa { 1.0 } else if !ca.is_empty() && (qa.contains(&ca) || ca.contains(&qa)) { 0.8 } else { 0.0 };
0.6 * t + 0.4 * a
}
fn fetch_json(http: &Http, url: &str) -> Result<Option<Value>> {
Ok(match http.get(url)? {
Some(b) => {
let v: Value = serde_json::from_str(&b)?;
if v.get("error").is_some() { anyhow::bail!("deezer: {}", v["error"]); }
Some(v)
}
None => None,
})
}
/// Looks up every artist + title not searched yet (`limit` per run).
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
crate::hitlisten::create_tables(db)?;
let todo: Vec<(String, String)> = db.conn.prepare(
"SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_charts WHERE chart != 'a40') s
WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title)
AND artist != '' AND title != ''")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
let now = Utc::now().to_rfc3339();
let mut done = 0;
for (artist, title) in todo.into_iter().take(limit) {
let (qa, qt) = query_parts(&artist, &title);
let query = format!("{qa} {qt}");
let url = format!("{BASE}/search?q={}&limit=10", urlencode(&query));
let found = match fetch_json(http, &url) {
Ok(v) => v.unwrap_or(Value::Null),
Err(e) => { log.push(format!("deezer search '{query}': {e}")); continue; }
};
let hits = found["data"].as_array().cloned().unwrap_or_default();
let best = hits.iter().map(|h| (score(h, &qa, &qt), h)).max_by(|a, b| a.0.total_cmp(&b.0));
let (best_id, best_score) = match best {
Some((s, h)) if s >= 0.6 => (h["id"].as_i64(), Some(s)),
Some((s, _)) => (None, Some(s)),
None => (None, None),
};
if let Some(id) = best_id {
if let Err(e) = track(db, http, id, &now) { log.push(format!("deezer track {id}: {e}")); }
}
// Every other strong match is another release of the same song (album, compilation,
// reissue): store those releases too, so cleaning can take the earliest release date.
for h in hits.iter().filter(|h| score(h, &qa, &qt) >= 0.9).take(6) {
if let Some(album_id) = h["album"]["id"].as_i64() {
if let Err(e) = album(db, http, album_id, &now) { log.push(format!("deezer album {album_id}: {e}")); }
}
}
db.conn.execute("INSERT OR REPLACE INTO deezer_searches VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![artist, title, query, hits.len() as i64, best_id, best_score, now, found.to_string()])?;
done += 1;
}
Ok(done)
}
/// Stores a track and its album (each fetched once).
fn track(db: &Db, http: &Http, id: i64, now: &str) -> Result<()> {
let have: Option<i64> = db.conn.query_row("SELECT album_id FROM deezer_tracks WHERE track_id = ?1", [id], |r| r.get(0)).optional()?;
let album_id = match have {
Some(a) => a,
None => {
let Some(t) = fetch_json(http, &format!("{BASE}/track/{id}"))? else { return Ok(()) };
let contributors: Vec<Value> = t["contributors"].as_array().cloned().unwrap_or_default().iter()
.map(|c| serde_json::json!({"name": c["name"], "role": c["role"], "id": c["id"]})).collect();
db.conn.execute("INSERT OR REPLACE INTO deezer_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![id, t["isrc"].as_str(), t["title"].as_str(), t["artist"]["name"].as_str(),
Value::Array(contributors).to_string(), t["release_date"].as_str(), t["duration"].as_i64(),
t["rank"].as_i64(), t["explicit_lyrics"].as_bool().map(|b| b as i64),
t["album"]["id"].as_i64(), now, t.to_string()])?;
t["album"]["id"].as_i64().unwrap_or(0)
}
};
if album_id != 0 { album(db, http, album_id, now)?; }
Ok(())
}
/// Stores a release once.
fn album(db: &Db, http: &Http, album_id: i64, now: &str) -> Result<()> {
let known: i64 = db.conn.query_row("SELECT count(*) FROM deezer_albums WHERE album_id = ?1", [album_id], |r| r.get(0))?;
if known > 0 { return Ok(()); }
if let Some(a) = fetch_json(http, &format!("{BASE}/album/{album_id}"))? {
let genres: Vec<&str> = a["genres"]["data"].as_array().map(|g| g.iter().filter_map(|x| x["name"].as_str()).collect()).unwrap_or_default();
db.conn.execute("INSERT OR REPLACE INTO deezer_albums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![album_id, a["title"].as_str(), a["upc"].as_str(), a["label"].as_str(), a["release_date"].as_str(),
a["record_type"].as_str(), genres.join(", "), a["nb_tracks"].as_i64(), now, a.to_string()])?;
}
Ok(())
}
fn urlencode(s: &str) -> String {
s.bytes().map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
b' ' => "%20".into(),
_ => format!("%{b:02X}"),
}).collect()
}
// ------------------------------------------------------------------ discographies
/// Every DR artist's releases from Deezer, for release dates and the historical candidate pool:
/// GET /search/artist?q={name} the Deezer artist: exact name match, most fans
/// GET /artist/{id}/albums every release (album, EP, single) with its date
pub fn create_discography_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS deezer_artists (
dr_artist_id TEXT PRIMARY KEY, deezer_artist_id INTEGER, name TEXT, fans INTEGER,
candidates INTEGER, fetched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS deezer_releases (
deezer_artist_id INTEGER NOT NULL, album_id INTEGER NOT NULL, title TEXT, release_date TEXT,
record_type TEXT, explicit INTEGER, fetched_at TEXT NOT NULL,
PRIMARY KEY (deezer_artist_id, album_id)
);
CREATE INDEX IF NOT EXISTS deezer_releases_date ON deezer_releases (release_date);",
)?;
Ok(())
}
pub fn discographies(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_discography_tables(db)?;
let todo: Vec<(String, String)> = db.conn.prepare(
"SELECT artist_id, name FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM deezer_artists) ORDER BY artist_id")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
let now = Utc::now().to_rfc3339();
let mut done = 0;
for (dr_id, name) in todo.into_iter().take(limit) {
let found = match fetch_json(http, &format!("{BASE}/search/artist?q={}&limit=10", urlencode(&name))) {
Ok(v) => v.unwrap_or(Value::Null),
Err(e) => { log.push(format!("deezer artist '{name}': {e}")); continue; }
};
let hits = found["data"].as_array().cloned().unwrap_or_default();
let best = hits.iter().filter(|h| key(h["name"].as_str().unwrap_or_default()) == key(&name))
.max_by_key(|h| h["nb_fan"].as_i64().unwrap_or(0));
let (did, fans) = (best.and_then(|h| h["id"].as_i64()), best.and_then(|h| h["nb_fan"].as_i64()));
if let Some(id) = did {
let have: i64 = db.conn.query_row("SELECT count(*) FROM deezer_releases WHERE deezer_artist_id = ?1", [id], |r| r.get(0))?;
if have == 0 {
let mut url = Some(format!("{BASE}/artist/{id}/albums?limit=100"));
let mut pages = 0;
while let Some(u) = url.take() {
let v = match fetch_json(http, &u) { Ok(Some(v)) => v, Ok(None) => break, Err(e) => { log.push(format!("deezer albums {id}: {e}")); break; } };
for a in v["data"].as_array().into_iter().flatten() {
db.conn.execute("INSERT OR REPLACE INTO deezer_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params![id, a["id"].as_i64(), a["title"].as_str(), a["release_date"].as_str(), a["record_type"].as_str(),
a["explicit_lyrics"].as_bool().map(|b| b as i64), now])?;
}
pages += 1;
url = v["next"].as_str().filter(|_| pages < 5).map(str::to_string);
}
}
}
db.conn.execute("INSERT OR REPLACE INTO deezer_artists VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![dr_id, did, name, fans, hits.len() as i64, now])?;
done += 1;
}
Ok(done)
}