thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00
Code Issues Activity

http: realistic UA; fall back to WebPage on probe failure

Browse source 
Replace bare 'archivr/0.1' user agent with a full Chrome 131 UA string
(archivr/0.1 token retained at the end) in both probe_url_kind and download.

More importantly, stop hard-failing when probe_url_kind returns an error.
Sites behind Cloudflare's managed JS challenge (e.g. Medium) return 403
before any header tuning can help — a plain HTTP client cannot pass the
challenge. Instead, log a warning and fall back to Source::WebPage so that
the SingleFile/Chromium path gets a chance; a real browser can solve the
challenge transparently.
This commit is contained in:
TheGeneralist 2026-07-11 14:07:56 +02:00
parent 2e8820a0da
commit 51d03f12e2
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
2 changed files with 8 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/archivr-core/src/capture.rs
View file

@ -900,14 +900,12 @@ pub fn perform_capture(
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
Ok(downloader::http::UrlKind::File) => {}
Err(e) => {
// Record a failed item using the pre-probe source_kind so
// failed_count increments and the item carries error_text.
let (probe_sk, probe_ek, _) = source_metadata(Source::Url);
let item = database::create_archive_run_item(
&conn, run.id, None, 0, locator, None, probe_sk, probe_ek,
)?;
let msg = format!("Failed to probe URL: {e}");
return Err(fail_run(&conn, &run, &item, &msg));
// Probe failed (e.g. Cloudflare JS challenge, 403, network
// issue). Fall back to treating the URL as an HTML page so
// that the SingleFile/Chromium path can try — a real browser
// can pass bot challenges that a plain HTTP client cannot.
eprintln!("warn: probe failed for {locator}, assuming HTML: {e}");
source = Source::WebPage;
}
}
}

4
crates/archivr-core/src/downloader/http.rs
View file

@ -20,7 +20,7 @@ pub enum UrlKind {
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10))
.user_agent("archivr/0.1")
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
.build()
.context("failed to build HTTP client")?;
@ -86,7 +86,7 @@ pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<Ur
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10))
.user_agent("archivr/0.1")
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
.build()
.context("failed to build HTTP client")?;