Song metadata: Deezer (ISRC, release date, label, type, genres, contributors) and MusicBrainz original release dates; data audit (airplay check)
Raw Deezer answers are kept so matches can be redone during cleaning. Every strong match's release is stored, so the earliest release date can be taken (Deezer often returns reissues). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
//! Original release dates from MusicBrainz, by ISRC: a recording's `first-release-date` is its
|
||||
//! earliest known release, where Deezer often gives a reissue or compilation instead (2Pac's
|
||||
//! "Changes" as 2009). MusicBrainz misses many new and Danish releases; for those Deezer's
|
||||
//! date is the original anyway. Cleaning takes the earliest date any source gives.
|
||||
//!
|
||||
//! GET https://musicbrainz.org/ws/2/isrc/{isrc}?fmt=json&inc=artist-credits
|
||||
//!
|
||||
//! MusicBrainz allows one request per second and asks for an identifying User-Agent.
|
||||
|
||||
use crate::{db::Db, http::Http};
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
use rusqlite::params;
|
||||
use serde_json::Value;
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS mb_isrc (
|
||||
isrc TEXT PRIMARY KEY,
|
||||
recordings INTEGER NOT NULL, -- 0: MusicBrainz doesn't know this ISRC
|
||||
first_release_date TEXT, -- earliest over the recordings with this ISRC
|
||||
recording_ids TEXT,
|
||||
fetched_at TEXT NOT NULL, raw TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
crate::deezer::create_tables(db)?;
|
||||
let todo: Vec<String> = db.conn.prepare(
|
||||
"SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != ''
|
||||
AND isrc NOT IN (SELECT isrc FROM mb_isrc)")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
let mut done = 0;
|
||||
for isrc in todo.into_iter().take(limit) {
|
||||
let body = match http.get(&format!("https://musicbrainz.org/ws/2/isrc/{isrc}?fmt=json&inc=artist-credits")) {
|
||||
Ok(b) => b,
|
||||
Err(e) => { log.push(format!("musicbrainz {isrc}: {e}")); continue; }
|
||||
};
|
||||
// 404 = MusicBrainz has no recording with this ISRC: stored as 0 recordings.
|
||||
let v: Value = body.as_deref().map(serde_json::from_str).transpose()?.unwrap_or(Value::Null);
|
||||
let recs = v["recordings"].as_array().cloned().unwrap_or_default();
|
||||
let first = recs.iter().filter_map(|r| r["first-release-date"].as_str()).filter(|d| !d.is_empty()).min().map(str::to_string);
|
||||
let ids: Vec<&str> = recs.iter().filter_map(|r| r["id"].as_str()).collect();
|
||||
db.conn.execute("INSERT OR REPLACE INTO mb_isrc VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![isrc, recs.len() as i64, first, ids.join(","), Utc::now().to_rfc3339(), v.to_string()])?;
|
||||
done += 1;
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
Reference in New Issue
Block a user