diff --git a/src/archive.rs b/src/archive.rs index 64dec81..28f646b 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -93,6 +93,8 @@ pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec { pub fn collect(db: &Db, http: &Http, log: &mut Vec) -> Result { create_tables(db)?; let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?; + // Pages that gave no plays get another try each run (an archive outage looks like an empty page). + db.conn.execute("DELETE FROM archive_pages WHERE plays = 0", [])?; let mut pages = 0; for ch in CHANNELS { // The CDX index is often overloaded (503): retry patiently. @@ -120,6 +122,10 @@ pub fn collect(db: &Db, http: &Http, log: &mut Vec) -> Result { Ok(h) => h.unwrap_or_default(), Err(e) => { log.push(format!("archive {url}: {e}")); continue; } }; + if html.is_empty() || html.contains("Internet Archive: Temporarily Offline") { + std::thread::sleep(std::time::Duration::from_secs(60)); + continue; + } let Some(c) = path.captures(url) else { continue }; let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue }; let plays = parse(&html, page_date);