Archive: skip Wayback outage pages, retry empty pages
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -93,6 +93,8 @@ pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec<Play> {
|
||||
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?;
|
||||
// Pages that gave no plays get another try each run (an archive outage looks like an empty page).
|
||||
db.conn.execute("DELETE FROM archive_pages WHERE plays = 0", [])?;
|
||||
let mut pages = 0;
|
||||
for ch in CHANNELS {
|
||||
// The CDX index is often overloaded (503): retry patiently.
|
||||
@@ -120,6 +122,10 @@ pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
Ok(h) => h.unwrap_or_default(),
|
||||
Err(e) => { log.push(format!("archive {url}: {e}")); continue; }
|
||||
};
|
||||
if html.is_empty() || html.contains("Internet Archive: Temporarily Offline") {
|
||||
std::thread::sleep(std::time::Duration::from_secs(60));
|
||||
continue;
|
||||
}
|
||||
let Some(c) = path.captures(url) else { continue };
|
||||
let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue };
|
||||
let plays = parse(&html, page_date);
|
||||
|
||||
Reference in New Issue
Block a user