New-add model (add = entering rotation after 4+ weeks out; per channel and any DR channel, 8-week window), site ranks by it; DR rotation-list PDF parser (glyph positions, channel by play profile) and Wayback Machine backfill: 29 weekly lists 2023-2026
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,343 @@
|
||||
//! DR's weekly rotation lists (PDF): which songs are in rotation, in which category, per channel.
|
||||
//!
|
||||
//! The PDF merges one page per channel. Each page has category headers (A, B, C, and on the
|
||||
//! P3 page also "Nye i power", "Ugens crush", "P3's uundgåelige") with artist + title lines
|
||||
//! under them, in one or several columns. Glyph positions are read from the PDF; a line belongs
|
||||
//! to the nearest header above it in its column. Which channel a page is gets decided from the
|
||||
//! data: the channel where the page's artists are played most (DR artist pages), so a change in
|
||||
//! page order doesn't mislabel anything. The P4 list is national and stored as channel "p4".
|
||||
//!
|
||||
//! airplay rotations [--reparse] parse every saved PDF into rotation_lists
|
||||
//! airplay rotations --wayback also fetch older lists archived by the Wayback Machine
|
||||
|
||||
use crate::{db::Db, http::Http};
|
||||
use anyhow::{anyhow, Result};
|
||||
use chrono::{Datelike, NaiveDate, Utc};
|
||||
use pdf_extract::{MediaBox, OutputDev, OutputError, Transform};
|
||||
use regex::Regex;
|
||||
use rusqlite::params;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS rotation_lists (
|
||||
list_date TEXT NOT NULL, -- the date in the PDF's file name (published, a Friday)
|
||||
iso_year INTEGER, iso_week INTEGER, -- 'UGE nn' printed on the list: the week it applies to
|
||||
channel TEXT NOT NULL, -- p3 | p4 (national, all regions) | p5 | p6beat | p8jazz
|
||||
category TEXT NOT NULL, -- A | B | C | or the list's own label (NYE I POWER, UGENS CRUSH, UUNDGÅELIGE)
|
||||
position INTEGER NOT NULL, -- order within the category
|
||||
artist TEXT NOT NULL, title TEXT NOT NULL,
|
||||
file TEXT NOT NULL, page INTEGER NOT NULL,
|
||||
channel_score REAL, -- how clearly the page matched its channel (share of plays)
|
||||
PRIMARY KEY (list_date, channel, category, position)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS rotation_wayback (
|
||||
snapshot TEXT PRIMARY KEY, pdf_url TEXT, fetched_at TEXT NOT NULL, status TEXT
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ glyphs → runs
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Run { pub page: u32, pub x: f64, pub y: f64, pub x_end: f64, pub size: f64, pub text: String }
|
||||
|
||||
#[derive(Default)]
|
||||
struct Glyphs { page: u32, width: f64, height: f64, glyphs: Vec<(u32, f64, f64, f64, f64, String)> }
|
||||
|
||||
impl OutputDev for Glyphs {
|
||||
fn begin_page(&mut self, page_num: u32, media_box: &MediaBox, _: Option<(f64, f64, f64, f64)>) -> Result<(), OutputError> {
|
||||
self.page = page_num;
|
||||
self.width = media_box.urx - media_box.llx;
|
||||
self.height = media_box.ury - media_box.lly;
|
||||
Ok(())
|
||||
}
|
||||
fn end_page(&mut self) -> Result<(), OutputError> { Ok(()) }
|
||||
fn output_character(&mut self, trm: &Transform, width: f64, _spacing: f64, font_size: f64, ch: &str) -> Result<(), OutputError> {
|
||||
let scale = (trm.m11 * trm.m11 + trm.m12 * trm.m12).sqrt();
|
||||
let size = font_size * scale;
|
||||
let (x, y) = (trm.m31, self.height - trm.m32);
|
||||
// Decoration: rotated text (the vertical "ROTATIONSLISTEN" frame, the tilted "P3 ROTATIONER"
|
||||
// background of older lists) and big letters along the page edges.
|
||||
// (m12 is the rotation; m21 alone is the skew of fake italics, which is kept)
|
||||
let rotated = trm.m12.abs() > 0.2 * scale.max(1e-9);
|
||||
let margin = size > 25.0 && (x < 0.06 * self.width || x > 0.94 * self.width || y < 0.06 * self.height || y > 0.94 * self.height);
|
||||
if !margin && !rotated { self.glyphs.push((self.page, x, y, width * size, size, ch.to_string())); }
|
||||
Ok(())
|
||||
}
|
||||
fn begin_word(&mut self) -> Result<(), OutputError> { Ok(()) }
|
||||
fn end_word(&mut self) -> Result<(), OutputError> { Ok(()) }
|
||||
fn end_line(&mut self) -> Result<(), OutputError> { Ok(()) }
|
||||
}
|
||||
|
||||
/// Text runs per page. Glyphs drawn twice at one spot (fake bold) count once; a run ends at a
|
||||
/// gap wider than ~1.6 times the font size (a column gap).
|
||||
pub fn runs(bytes: &[u8]) -> Result<Vec<Run>> {
|
||||
let doc = pdf_extract::Document::load_mem(bytes)?;
|
||||
let mut g = Glyphs::default();
|
||||
pdf_extract::output_doc(&doc, &mut g)?;
|
||||
let mut glyphs = g.glyphs;
|
||||
glyphs.sort_by(|a, b| a.0.cmp(&b.0).then(a.2.total_cmp(&b.2)).then(a.1.total_cmp(&b.1)));
|
||||
let mut lines: Vec<Vec<(u32, f64, f64, f64, f64, String)>> = Vec::new();
|
||||
for gl in glyphs {
|
||||
match lines.last_mut() {
|
||||
Some(l) if l[0].0 == gl.0 && (gl.2 - l[0].2).abs() < l[0].4.min(gl.4).max(1.0) * 0.35 => l.push(gl),
|
||||
_ => lines.push(vec![gl]),
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for mut l in lines {
|
||||
l.sort_by(|a, b| a.1.total_cmp(&b.1));
|
||||
l.dedup_by(|b, a| a.5 == b.5 && (a.1 - b.1).abs() < 0.5);
|
||||
let mut cur: Option<Run> = None;
|
||||
for (page, x, y, w, size, ch) in l {
|
||||
if let Some(r) = cur.as_mut() {
|
||||
let same_size = (size - r.size).abs() < 0.5;
|
||||
if same_size && x - r.x_end < size * 0.45 { r.text.push_str(&ch); r.x_end = r.x_end.max(x + w); continue; }
|
||||
if same_size && x - r.x_end < size * 1.6 { r.text.push(' '); r.text.push_str(&ch); r.x_end = x + w; continue; }
|
||||
out.push(cur.take().unwrap());
|
||||
}
|
||||
cur = Some(Run { page, x, y, x_end: x + w, size, text: ch });
|
||||
}
|
||||
if let Some(r) = cur { out.push(r); }
|
||||
}
|
||||
for r in &mut out { r.text = r.text.split_whitespace().collect::<Vec<_>>().join(" "); }
|
||||
out.retain(|r| !r.text.is_empty());
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn dump(path: &std::path::Path) -> Result<()> {
|
||||
for r in runs(&std::fs::read(path)?)? {
|
||||
println!("p{} y{:7.1} x{:6.1}-{:6.1} s{:4.1} | {}", r.page, r.y, r.x, r.x_end, r.size, r.text);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ runs → entries
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Entry { pub page: u32, pub category: String, pub position: usize, pub artist: String, pub title: String }
|
||||
|
||||
pub struct Parsed { pub week: Option<u32>, pub entries: Vec<Entry> }
|
||||
|
||||
pub fn parse(bytes: &[u8]) -> Result<Parsed> {
|
||||
let runs = runs(bytes)?;
|
||||
let week_re = Regex::new(r"UGE\s*(\d{1,2})")?;
|
||||
let week = runs.iter().find_map(|r| week_re.captures(&r.text).and_then(|c| c[1].parse().ok()));
|
||||
let is_cat = |t: &str| matches!(t, "A" | "B" | "C");
|
||||
let is_label = |r: &Run| r.size >= 12.0 && r.size < 40.0 && r.text.chars().count() >= 4 && !is_cat(&r.text)
|
||||
&& !r.text.starts_with("UGE") && !r.text.starts_with("E ") && !r.text.contains("ROTATIONSLISTEN");
|
||||
let mut entries = Vec::new();
|
||||
let pages: Vec<u32> = { let mut p: Vec<u32> = runs.iter().map(|r| r.page).collect(); p.dedup(); p };
|
||||
for page in pages {
|
||||
let pr: Vec<&Run> = runs.iter().filter(|r| r.page == page).collect();
|
||||
// Headers: category letters and labelled sections.
|
||||
let headers: Vec<(&Run, String)> = pr.iter().filter(|r| (r.size >= 12.0 && is_cat(&r.text)) || is_label(r))
|
||||
.map(|r| (*r, if is_cat(&r.text) { r.text.clone() } else { label(&r.text) })).collect();
|
||||
// "P3's uundgåelige": the two big runs after the label are artist and title.
|
||||
if let Some((h, _)) = headers.iter().find(|(r, _)| r.text.to_uppercase().contains("UUNDGÅELIGE")) {
|
||||
let big: Vec<&&Run> = pr.iter().filter(|r| r.size >= 40.0 && r.y > h.y && !is_cat(&r.text)).take(2).collect();
|
||||
if big.len() == 2 {
|
||||
entries.push(Entry { page, category: "UUNDGÅELIGE".into(), position: 1, artist: big[0].text.clone(), title: big[1].text.clone() });
|
||||
}
|
||||
}
|
||||
let body: Vec<&Run> = pr.iter().copied().filter(|r| r.size < 12.0).collect();
|
||||
// Columns start at the category letters (A, B, C side by side on the P3 summary page).
|
||||
let mut cols: Vec<f64> = headers.iter().filter(|(r, _)| is_cat(&r.text)).map(|(r, _)| r.x).collect();
|
||||
cols.sort_by(f64::total_cmp);
|
||||
cols.dedup_by(|b, a| (*b - *a).abs() < 40.0);
|
||||
let col_of = |x: f64| cols.iter().rposition(|c| *c <= x + 20.0).unwrap_or(0);
|
||||
// Lines: same y and same column; the header: the lowest one above, in that column.
|
||||
let mut lines: Vec<(usize, Vec<&Run>)> = Vec::new();
|
||||
let mut sorted = body.clone();
|
||||
sorted.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.x.total_cmp(&b.x)));
|
||||
for r in sorted {
|
||||
let c = col_of(r.x);
|
||||
match lines.iter_mut().find(|(lc, l)| *lc == c && (l[0].y - r.y).abs() < 2.0) {
|
||||
Some((_, l)) => l.push(r),
|
||||
None => lines.push((c, vec![r])),
|
||||
}
|
||||
}
|
||||
let mut count: HashMap<usize, usize> = HashMap::new();
|
||||
for (c, mut l) in lines {
|
||||
l.sort_by(|a, b| a.x.total_cmp(&b.x));
|
||||
if l.len() < 2 { continue; } // a lone run: not an artist + title pair
|
||||
let Some((hi, (_, cat))) = headers.iter().enumerate()
|
||||
.filter(|(_, (h, _))| col_of(h.x) == c && h.y < l[0].y + 1.0)
|
||||
.max_by(|a, b| a.1 .0.y.total_cmp(&b.1 .0.y)) else { continue };
|
||||
let n = count.entry(hi).or_insert(0);
|
||||
*n += 1;
|
||||
entries.push(Entry { page, category: cat.clone(), position: *n, artist: l[0].text.clone(),
|
||||
title: l[1..].iter().map(|r| r.text.as_str()).collect::<Vec<_>>().join(" ") });
|
||||
}
|
||||
}
|
||||
Ok(Parsed { week, entries })
|
||||
}
|
||||
|
||||
fn label(t: &str) -> String {
|
||||
let u = t.to_uppercase();
|
||||
if u.contains("CRUSH") { "UGENS CRUSH".into() }
|
||||
else if u.contains("NYE I POWER") { "NYE I POWER".into() }
|
||||
else if u.contains("RECURRENT") { "RECURRENTS".into() }
|
||||
else if u.contains("ROTATIONER") || u.contains("ROTATIONSLISTE") { "LIST".into() }
|
||||
else { u }
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ pages → channels
|
||||
|
||||
const LIST_CHANNELS: [&str; 5] = ["p3", "p4", "p5", "p6beat", "p8jazz"];
|
||||
|
||||
/// Share of each list channel in the plays of the page's artists (DR artist pages, all weeks).
|
||||
fn channel_scores(db: &Db, artists: &[&str]) -> Result<[f64; 5]> {
|
||||
let key = crate::model::panel::key;
|
||||
let mut by_name: HashMap<String, Vec<String>> = HashMap::new();
|
||||
for r in db.conn.prepare("SELECT artist_id, name FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||||
let (id, n) = r?;
|
||||
by_name.entry(key(&n)).or_default().push(id);
|
||||
}
|
||||
let mut st = db.conn.prepare("SELECT channel, sum(plays) FROM dr_artist_weeks WHERE artist_id = ?1 GROUP BY channel")?;
|
||||
let mut score = [0.0; 5];
|
||||
for a in artists {
|
||||
let first = a.split(&['[', ','][..]).next().unwrap_or(a).split(" feat").next().unwrap_or(a).split(" ft.").next().unwrap_or(a)
|
||||
.split(" & ").next().unwrap_or(a).split(" x ").next().unwrap_or(a).trim();
|
||||
let Some(ids) = by_name.get(&key(first)) else { continue };
|
||||
let mut per = [0.0; 5];
|
||||
for id in ids {
|
||||
for r in st.query_map([id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? {
|
||||
let (ch, n) = r?;
|
||||
let k = if ch.starts_with("p4") && ch != "p4" { Some(1) } else { LIST_CHANNELS.iter().position(|c| *c == ch) };
|
||||
// the ten regional P4s count as one channel
|
||||
if let Some(k) = k { per[k] += n as f64 / if k == 1 { 10.0 } else { 1.0 }; }
|
||||
}
|
||||
}
|
||||
let tot: f64 = per.iter().sum();
|
||||
if tot > 0.0 { for k in 0..5 { score[k] += per[k] / tot; } }
|
||||
}
|
||||
Ok(score)
|
||||
}
|
||||
|
||||
/// Parses one PDF and stores its rows (a list date already stored is left alone unless `replace`).
|
||||
pub fn store(db: &Db, file: &std::path::Path, replace: bool, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
let name = file.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_string();
|
||||
let date = Regex::new(r"^(\d{2})-(\d{2})-(\d{4})")?.captures(&name)
|
||||
.and_then(|c| NaiveDate::from_ymd_opt(c[3].parse().ok()?, c[2].parse().ok()?, c[1].parse().ok()?))
|
||||
.or_else(|| Regex::new(r"^(\d{4})-(\d{2})-(\d{2})").ok()?.captures(&name)
|
||||
.and_then(|c| NaiveDate::from_ymd_opt(c[1].parse().ok()?, c[2].parse().ok()?, c[3].parse().ok()?)))
|
||||
.ok_or_else(|| anyhow!("{name}: no date in the file name"))?;
|
||||
let have: i64 = db.conn.query_row("SELECT count(*) FROM rotation_lists WHERE list_date = ?1", [date.to_string()], |r| r.get(0))?;
|
||||
if have > 0 && !replace { return Ok(0); }
|
||||
let parsed = parse(&std::fs::read(file)?)?;
|
||||
// ISO week of the list: "UGE nn" near the list date (the list is for the coming week).
|
||||
let iso = parsed.week.map(|w| {
|
||||
let y = date.iso_week().year();
|
||||
let y = if w < 10 && date.iso_week().week() > 40 { y + 1 } else { y };
|
||||
(y, w)
|
||||
});
|
||||
let mut pages: Vec<u32> = parsed.entries.iter().map(|e| e.page).collect();
|
||||
pages.sort(); pages.dedup();
|
||||
let mut channel_of: HashMap<u32, (String, f64)> = HashMap::new();
|
||||
let mut taken: Vec<usize> = Vec::new();
|
||||
// Most distinctive pages first, each channel used once (P3 may span two pages: the summary page with "uundgåelige").
|
||||
let mut scored: Vec<(u32, [f64; 5], f64)> = Vec::new();
|
||||
for &pg in &pages {
|
||||
let artists: Vec<&str> = parsed.entries.iter().filter(|e| e.page == pg).map(|e| e.artist.as_str()).collect();
|
||||
let s = channel_scores(db, &artists)?;
|
||||
let tot: f64 = s.iter().sum();
|
||||
scored.push((pg, s, tot));
|
||||
}
|
||||
scored.sort_by(|a, b| { let m = |x: &[f64; 5], t: f64| x.iter().cloned().fold(0.0, f64::max) / t.max(1e-9); m(&b.1, b.2).total_cmp(&m(&a.1, a.2)) });
|
||||
for (pg, s, tot) in &scored {
|
||||
let mut order: Vec<usize> = (0..5).collect();
|
||||
order.sort_by(|&a, &b| s[b].total_cmp(&s[a]));
|
||||
let k = order.iter().copied().find(|k| !taken.contains(k) || (*k == 0)).unwrap_or(order[0]);
|
||||
if k != 0 { taken.push(k); }
|
||||
channel_of.insert(*pg, (LIST_CHANNELS[k].to_string(), s[k] / tot.max(1e-9)));
|
||||
if s[k] / tot.max(1e-9) < 0.4 { log.push(format!("{name} page {pg}: weak channel match ({} {:.2})", LIST_CHANNELS[k], s[k] / tot.max(1e-9))); }
|
||||
}
|
||||
let tx = db.conn.unchecked_transaction()?;
|
||||
tx.execute("DELETE FROM rotation_lists WHERE list_date = ?1", [date.to_string()])?;
|
||||
let mut n = 0;
|
||||
for e in &parsed.entries {
|
||||
let (ch, score) = channel_of.get(&e.page).cloned().unwrap_or(("unknown".into(), 0.0));
|
||||
// The P3 summary page repeats categories A/B/C of a sub-list; keep its own labels apart.
|
||||
let cat = if ch == "p3" && pages.len() > 5 && e.page == pages[0] && matches!(e.category.as_str(), "A" | "B" | "C") { format!("P3 SIDE {}", e.category) } else { e.category.clone() };
|
||||
tx.execute("INSERT OR REPLACE INTO rotation_lists VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
|
||||
params![date.to_string(), iso.map(|x| x.0), iso.map(|x| x.1), ch, cat, e.position as i64, e.artist, e.title, name, e.page as i64, score])?;
|
||||
n += 1;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Parses every PDF in `dir`.
|
||||
pub fn store_all(db: &Db, dir: &std::path::Path, replace: bool) -> Result<()> {
|
||||
let mut log = Vec::new();
|
||||
let mut files: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "pdf")).collect();
|
||||
files.sort();
|
||||
for f in files {
|
||||
match store(db, &f, replace, &mut log) {
|
||||
Ok(n) => println!("{}: {n} rows", f.display()),
|
||||
Err(e) => println!("{}: {e}", f.display()),
|
||||
}
|
||||
}
|
||||
for l in log { println!(" {l}"); }
|
||||
let mut st = db.conn.prepare("SELECT list_date, channel, category, count(*), round(avg(channel_score), 2) FROM rotation_lists GROUP BY 1, 2, 3 ORDER BY 1, 2, 3")?;
|
||||
for r in st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, f64>(4)?)))? {
|
||||
let (d, c, k, n, s) = r?;
|
||||
println!(" {d} {c:8} {k:14} {n:3} (channel match {s})");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Wayback Machine
|
||||
|
||||
/// Older lists: every archived copy of DR's rotation page links that week's PDF; fetch the PDF
|
||||
/// from DR if it's still there, else from the archive. Each snapshot is tried once.
|
||||
pub fn wayback(db: &Db, http: &Http, dir: &std::path::Path, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
let page = "www.dr.dk/om-dr/nyheder/musik-i-rotation-paa-drs-radiokanaler-netop-nu";
|
||||
let cdx = http.get(&format!("https://web.archive.org/cdx/search/cdx?url={page}&output=json&fl=timestamp,statuscode&filter=statuscode:200&collapse=digest"))?
|
||||
.ok_or_else(|| anyhow!("no CDX answer"))?;
|
||||
let rows: Vec<Vec<String>> = serde_json::from_str(&cdx).map_err(|e| anyhow!("CDX: {e}"))?;
|
||||
// The link has moved over the years: www.dr.dk/static/documents/... (2023-24), asset.dr.dk/... (later).
|
||||
let pdf_re = Regex::new(r#"https?://(?:asset\.dr\.dk|www\.dr\.dk/static)/[^"'\s<>]+?rotation[^"'\s<>]*?\.pdf"#)?;
|
||||
let path_date = Regex::new(r"/(\d{4})/(\d{2})/(\d{2})/")?;
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let mut saved = 0;
|
||||
for row in rows.iter().skip(1) {
|
||||
let ts = &row[0];
|
||||
let done: i64 = db.conn.query_row("SELECT count(*) FROM rotation_wayback WHERE snapshot = ?1", [ts], |r| r.get(0))?;
|
||||
if done > 0 { continue; }
|
||||
let html = http.get(&format!("https://web.archive.org/web/{ts}id_/https://{page}"))?.unwrap_or_default();
|
||||
let url = pdf_re.find(&html).map(|m| m.as_str().replace("http://", "https://"));
|
||||
let status = match &url {
|
||||
None => "no pdf link".to_string(),
|
||||
Some(u) => {
|
||||
// Name the file by its publication date (dd-mm-yyyy first, which `store` reads).
|
||||
let base = u.rsplit('/').next().unwrap_or("rotation.pdf");
|
||||
let name = match path_date.captures(u) { Some(c) => format!("{}-{}-{}-{base}", &c[3], &c[2], &c[1]), None => base.to_string() };
|
||||
let file = dir.join(name);
|
||||
if file.exists() { "already saved".to_string() } else {
|
||||
let bytes = match http.get_bytes(u) {
|
||||
Ok(Some(b)) if b.starts_with(b"%PDF") => Some(b),
|
||||
_ => http.get_bytes(&format!("https://web.archive.org/web/{ts}id_/{u}")).ok().flatten().filter(|b| b.starts_with(b"%PDF")),
|
||||
};
|
||||
match bytes {
|
||||
Some(b) => {
|
||||
std::fs::write(&file, &b)?;
|
||||
db.conn.execute("INSERT OR IGNORE INTO dr_rotation_pdfs VALUES (?1, ?2, ?3)", params![u, file.to_string_lossy(), Utc::now().to_rfc3339()])?;
|
||||
saved += 1;
|
||||
"saved".to_string()
|
||||
}
|
||||
None => { log.push(format!("wayback {ts}: {u} not retrievable")); "pdf not retrievable".to_string() }
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
db.conn.execute("INSERT OR REPLACE INTO rotation_wayback VALUES (?1, ?2, ?3, ?4)", params![ts, url, Utc::now().to_rfc3339(), status])?;
|
||||
println!("{ts}: {status} {}", url.unwrap_or_default());
|
||||
}
|
||||
Ok(saved)
|
||||
}
|
||||
Reference in New Issue
Block a user