Archive: skip Wayback outage pages, retry empty pages

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 04:21:57 +00:00
co-authored by Claude Opus 5.5
parent 650a6a0f38
commit 73366acd07
+6
View File
@@ -93,6 +93,8 @@ pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec<Play> {
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> { pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?; create_tables(db)?;
let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?; let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?;
// Pages that gave no plays get another try each run (an archive outage looks like an empty page).
db.conn.execute("DELETE FROM archive_pages WHERE plays = 0", [])?;
let mut pages = 0; let mut pages = 0;
for ch in CHANNELS { for ch in CHANNELS {
// The CDX index is often overloaded (503): retry patiently. // The CDX index is often overloaded (503): retry patiently.
@@ -120,6 +122,10 @@ pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
Ok(h) => h.unwrap_or_default(), Ok(h) => h.unwrap_or_default(),
Err(e) => { log.push(format!("archive {url}: {e}")); continue; } Err(e) => { log.push(format!("archive {url}: {e}")); continue; }
}; };
if html.is_empty() || html.contains("Internet Archive: Temporarily Offline") {
std::thread::sleep(std::time::Duration::from_secs(60));
continue;
}
let Some(c) = path.captures(url) else { continue }; let Some(c) = path.captures(url) else { continue };
let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue }; let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue };
let plays = parse(&html, page_date); let plays = parse(&html, page_date);