diff --git a/crates/archivr-core/src/capture.rs b/crates/archivr-core/src/capture.rs index 507163c..8004537 100644 --- a/crates/archivr-core/src/capture.rs +++ b/crates/archivr-core/src/capture.rs @@ -900,14 +900,12 @@ pub fn perform_capture( Ok(downloader::http::UrlKind::Html) => source = Source::WebPage, Ok(downloader::http::UrlKind::File) => {} Err(e) => { - // Record a failed item using the pre-probe source_kind so - // failed_count increments and the item carries error_text. - let (probe_sk, probe_ek, _) = source_metadata(Source::Url); - let item = database::create_archive_run_item( - &conn, run.id, None, 0, locator, None, probe_sk, probe_ek, - )?; - let msg = format!("Failed to probe URL: {e}"); - return Err(fail_run(&conn, &run, &item, &msg)); + // Probe failed (e.g. Cloudflare JS challenge, 403, network + // issue). Fall back to treating the URL as an HTML page so + // that the SingleFile/Chromium path can try — a real browser + // can pass bot challenges that a plain HTTP client cannot. + eprintln!("warn: probe failed for {locator}, assuming HTML: {e}"); + source = Source::WebPage; } } } diff --git a/crates/archivr-core/src/downloader/http.rs b/crates/archivr-core/src/downloader/http.rs index 282910a..24847d7 100644 --- a/crates/archivr-core/src/downloader/http.rs +++ b/crates/archivr-core/src/downloader/http.rs @@ -20,7 +20,7 @@ pub enum UrlKind { pub fn probe_url_kind(url: &str, cookies: &HashMap) -> Result { let client = reqwest::blocking::Client::builder() .redirect(reqwest::redirect::Policy::limited(10)) - .user_agent("archivr/0.1") + .user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1") .build() .context("failed to build HTTP client")?; @@ -86,7 +86,7 @@ pub fn probe_url_kind(url: &str, cookies: &HashMap) -> Result) -> Result<(String, String, Option)> { let client = reqwest::blocking::Client::builder() .redirect(reqwest::redirect::Policy::limited(10)) - .user_agent("archivr/0.1") + .user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1") .build() .context("failed to build HTTP client")?;