Archive: skip Wayback outage pages, retry empty pages
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -93,6 +93,8 @@ pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec<Play> {
|
|||||||
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||||
create_tables(db)?;
|
create_tables(db)?;
|
||||||
let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?;
|
let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?;
|
||||||
|
// Pages that gave no plays get another try each run (an archive outage looks like an empty page).
|
||||||
|
db.conn.execute("DELETE FROM archive_pages WHERE plays = 0", [])?;
|
||||||
let mut pages = 0;
|
let mut pages = 0;
|
||||||
for ch in CHANNELS {
|
for ch in CHANNELS {
|
||||||
// The CDX index is often overloaded (503): retry patiently.
|
// The CDX index is often overloaded (503): retry patiently.
|
||||||
@@ -120,6 +122,10 @@ pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
|||||||
Ok(h) => h.unwrap_or_default(),
|
Ok(h) => h.unwrap_or_default(),
|
||||||
Err(e) => { log.push(format!("archive {url}: {e}")); continue; }
|
Err(e) => { log.push(format!("archive {url}: {e}")); continue; }
|
||||||
};
|
};
|
||||||
|
if html.is_empty() || html.contains("Internet Archive: Temporarily Offline") {
|
||||||
|
std::thread::sleep(std::time::Duration::from_secs(60));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let Some(c) = path.captures(url) else { continue };
|
let Some(c) = path.captures(url) else { continue };
|
||||||
let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue };
|
let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue };
|
||||||
let plays = parse(&html, page_date);
|
let plays = parse(&html, page_date);
|
||||||
|
|||||||
Reference in New Issue
Block a user