mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
feat: uBlock Origin Lite integration for ad-blocking during WebPage captures
- singlefile.rs: when ARCHIVR_UBLOCK=true and ARCHIVR_UBLOCK_EXT is set,
archivr owns Chrome's lifecycle (--headless=new, --remote-debugging-port,
--load-extension); single-file connects via --browser-server instead of
launching its own Chrome. Falls back to old behaviour with ublock_skipped=true
when the ext path is missing or invalid.
- capture.rs: thread ublock_skipped through CaptureResult
- database.rs: add notes_json TEXT column to capture_jobs (DDL + idempotent
ALTER TABLE migration); update_capture_job_status gains notes_json param
- archive.rs: expose notes_json in CaptureJobSummary
- routes.rs: store {"ublock_skipped":true} in notes_json on completed captures
- ToastStack.jsx: warning toast variant (toast--warning) with Details expander
and Ignore button
- CaptureDialog.jsx: fire warning toast when poll result has ublock_skipped
- App.jsx: sessionStorage-backed Ignore suppression for ublock warnings
- styles.css: .toast--warning (amber left border) + .toast-warning-detail
- flake.nix: ublockLite derivation fetches uBOLite_2026.705.2152.chromium.zip
(pinned SHA256) from uBlockOrigin/uBOL-home; sets ARCHIVR_UBLOCK_EXT in both
archivr and archivr-server wrappers
Env vars:
ARCHIVR_UBLOCK=true (default) — enable uBlock during WebPage captures
ARCHIVR_UBLOCK_EXT — path to unpacked uBOL extension dir (set by Nix)
This commit is contained in:
parent
dae61e585d
commit
8a90259a76
14 changed files with 546 additions and 226 deletions
|
|
@ -69,6 +69,7 @@ pub struct CaptureJobSummary {
|
||||||
pub run_uid: Option<String>,
|
pub run_uid: Option<String>,
|
||||||
pub status: String,
|
pub status: String,
|
||||||
pub error_text: Option<String>,
|
pub error_text: Option<String>,
|
||||||
|
pub notes_json: Option<String>,
|
||||||
pub created_at: String,
|
pub created_at: String,
|
||||||
pub updated_at: String,
|
pub updated_at: String,
|
||||||
}
|
}
|
||||||
|
|
@ -341,6 +342,7 @@ pub fn get_capture_job(
|
||||||
run_uid: r.run_uid,
|
run_uid: r.run_uid,
|
||||||
status: r.status,
|
status: r.status,
|
||||||
error_text: r.error_text,
|
error_text: r.error_text,
|
||||||
|
notes_json: r.notes_json,
|
||||||
created_at: r.created_at,
|
created_at: r.created_at,
|
||||||
updated_at: r.updated_at,
|
updated_at: r.updated_at,
|
||||||
}))
|
}))
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,9 @@ pub enum Source {
|
||||||
pub struct CaptureResult {
|
pub struct CaptureResult {
|
||||||
pub run_uid: String,
|
pub run_uid: String,
|
||||||
pub status: String,
|
pub status: String,
|
||||||
|
/// `true` when uBlock was requested but the extension path was not found.
|
||||||
|
/// The capture succeeded without ad-blocking; the UI should warn the user.
|
||||||
|
pub ublock_skipped: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
|
|
@ -991,6 +994,7 @@ pub fn perform_capture(
|
||||||
return Ok(CaptureResult {
|
return Ok(CaptureResult {
|
||||||
run_uid: run.run_uid.clone(),
|
run_uid: run.run_uid.clone(),
|
||||||
status: "completed".to_string(),
|
status: "completed".to_string(),
|
||||||
|
ublock_skipped: false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -1133,6 +1137,7 @@ pub fn perform_capture(
|
||||||
return Ok(CaptureResult {
|
return Ok(CaptureResult {
|
||||||
run_uid: run.run_uid.clone(),
|
run_uid: run.run_uid.clone(),
|
||||||
status: "completed".to_string(),
|
status: "completed".to_string(),
|
||||||
|
ublock_skipped: result.ublock_skipped,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -1187,6 +1192,7 @@ pub fn perform_capture(
|
||||||
return Ok(CaptureResult {
|
return Ok(CaptureResult {
|
||||||
run_uid: run.run_uid.clone(),
|
run_uid: run.run_uid.clone(),
|
||||||
status: "completed".to_string(),
|
status: "completed".to_string(),
|
||||||
|
ublock_skipped: false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -1342,6 +1348,7 @@ pub fn perform_capture(
|
||||||
Ok(CaptureResult {
|
Ok(CaptureResult {
|
||||||
run_uid: run.run_uid.clone(),
|
run_uid: run.run_uid.clone(),
|
||||||
status: "completed".to_string(),
|
status: "completed".to_string(),
|
||||||
|
ublock_skipped: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,7 @@ pub struct CaptureJobRecord {
|
||||||
pub run_uid: Option<String>,
|
pub run_uid: Option<String>,
|
||||||
pub status: String,
|
pub status: String,
|
||||||
pub error_text: Option<String>,
|
pub error_text: Option<String>,
|
||||||
|
pub notes_json: Option<String>,
|
||||||
pub created_at: String,
|
pub created_at: String,
|
||||||
pub updated_at: String,
|
pub updated_at: String,
|
||||||
}
|
}
|
||||||
|
|
@ -308,6 +309,7 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> {
|
||||||
run_uid TEXT,
|
run_uid TEXT,
|
||||||
status TEXT NOT NULL CHECK (status IN ('pending','running','completed','failed')) DEFAULT 'pending',
|
status TEXT NOT NULL CHECK (status IN ('pending','running','completed','failed')) DEFAULT 'pending',
|
||||||
error_text TEXT,
|
error_text TEXT,
|
||||||
|
notes_json TEXT,
|
||||||
created_at TEXT NOT NULL,
|
created_at TEXT NOT NULL,
|
||||||
updated_at TEXT NOT NULL
|
updated_at TEXT NOT NULL
|
||||||
);
|
);
|
||||||
|
|
@ -393,6 +395,10 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> {
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Migration: add notes_json column to existing capture_jobs tables.
|
||||||
|
// Silently ignored when the column already exists (idempotent).
|
||||||
|
let _ = conn.execute("ALTER TABLE capture_jobs ADD COLUMN notes_json TEXT", []);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1142,19 +1148,21 @@ pub fn create_capture_job(conn: &Connection, archive_id: &str) -> Result<String>
|
||||||
Ok(job_uid)
|
Ok(job_uid)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Updates the status (and optionally run_uid / error_text) of a capture job.
|
/// Updates the status (and optionally run_uid / error_text / notes_json) of a capture job.
|
||||||
pub fn update_capture_job_status(
|
pub fn update_capture_job_status(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
job_uid: &str,
|
job_uid: &str,
|
||||||
status: &str,
|
status: &str,
|
||||||
run_uid: Option<&str>,
|
run_uid: Option<&str>,
|
||||||
error_text: Option<&str>,
|
error_text: Option<&str>,
|
||||||
|
notes_json: Option<&str>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let now = now_timestamp();
|
let now = now_timestamp();
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE capture_jobs SET status = ?1, run_uid = COALESCE(?2, run_uid),
|
"UPDATE capture_jobs SET status = ?1, run_uid = COALESCE(?2, run_uid),
|
||||||
error_text = ?3, updated_at = ?4 WHERE job_uid = ?5",
|
error_text = ?3, notes_json = COALESCE(?4, notes_json), updated_at = ?5
|
||||||
rusqlite::params![status, run_uid, error_text, now, job_uid],
|
WHERE job_uid = ?6",
|
||||||
|
rusqlite::params![status, run_uid, error_text, notes_json, now, job_uid],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -1162,7 +1170,7 @@ pub fn update_capture_job_status(
|
||||||
/// Returns a capture job by uid.
|
/// Returns a capture job by uid.
|
||||||
pub fn get_capture_job(conn: &Connection, job_uid: &str) -> Result<Option<CaptureJobRecord>> {
|
pub fn get_capture_job(conn: &Connection, job_uid: &str) -> Result<Option<CaptureJobRecord>> {
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
"SELECT job_uid, archive_id, run_uid, status, error_text, created_at, updated_at
|
"SELECT job_uid, archive_id, run_uid, status, error_text, notes_json, created_at, updated_at
|
||||||
FROM capture_jobs WHERE job_uid = ?1",
|
FROM capture_jobs WHERE job_uid = ?1",
|
||||||
[job_uid],
|
[job_uid],
|
||||||
|row| {
|
|row| {
|
||||||
|
|
@ -1172,8 +1180,9 @@ pub fn get_capture_job(conn: &Connection, job_uid: &str) -> Result<Option<Captur
|
||||||
run_uid: row.get(2)?,
|
run_uid: row.get(2)?,
|
||||||
status: row.get(3)?,
|
status: row.get(3)?,
|
||||||
error_text: row.get(4)?,
|
error_text: row.get(4)?,
|
||||||
created_at: row.get(5)?,
|
notes_json: row.get(5)?,
|
||||||
updated_at: row.get(6)?,
|
created_at: row.get(6)?,
|
||||||
|
updated_at: row.get(7)?,
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -2806,8 +2815,8 @@ mod tests {
|
||||||
fn capture_job_status_transitions() {
|
fn capture_job_status_transitions() {
|
||||||
let conn = conn();
|
let conn = conn();
|
||||||
let job_uid = create_capture_job(&conn, "test").unwrap();
|
let job_uid = create_capture_job(&conn, "test").unwrap();
|
||||||
update_capture_job_status(&conn, &job_uid, "running", None, None).unwrap();
|
update_capture_job_status(&conn, &job_uid, "running", None, None, None).unwrap();
|
||||||
update_capture_job_status(&conn, &job_uid, "completed", Some("run_abc"), None).unwrap();
|
update_capture_job_status(&conn, &job_uid, "completed", Some("run_abc"), None, None).unwrap();
|
||||||
let job = get_capture_job(&conn, &job_uid).unwrap().unwrap();
|
let job = get_capture_job(&conn, &job_uid).unwrap().unwrap();
|
||||||
assert_eq!(job.status, "completed");
|
assert_eq!(job.status, "completed");
|
||||||
assert_eq!(job.run_uid.as_deref(), Some("run_abc"));
|
assert_eq!(job.run_uid.as_deref(), Some("run_abc"));
|
||||||
|
|
@ -2819,7 +2828,7 @@ mod tests {
|
||||||
|
|
||||||
// Simulate an in-progress capture_job (run_uid still NULL — common crash case).
|
// Simulate an in-progress capture_job (run_uid still NULL — common crash case).
|
||||||
let uid = create_capture_job(&conn, "test").unwrap();
|
let uid = create_capture_job(&conn, "test").unwrap();
|
||||||
update_capture_job_status(&conn, &uid, "running", None, None).unwrap();
|
update_capture_job_status(&conn, &uid, "running", None, None, None).unwrap();
|
||||||
|
|
||||||
// Simulate an in-progress archive_run and item with no associated capture_job
|
// Simulate an in-progress archive_run and item with no associated capture_job
|
||||||
// (covers the case where run_uid was never written back before the crash).
|
// (covers the case where run_uid was never written back before the crash).
|
||||||
|
|
@ -3177,7 +3186,7 @@ mod tests {
|
||||||
fn has_active_capture_jobs_true_for_running() {
|
fn has_active_capture_jobs_true_for_running() {
|
||||||
let conn = conn();
|
let conn = conn();
|
||||||
let uid = create_capture_job(&conn, "test").unwrap();
|
let uid = create_capture_job(&conn, "test").unwrap();
|
||||||
update_capture_job_status(&conn, &uid, "running", None, None).unwrap();
|
update_capture_job_status(&conn, &uid, "running", None, None, None).unwrap();
|
||||||
assert!(has_active_capture_jobs(&conn).unwrap());
|
assert!(has_active_capture_jobs(&conn).unwrap());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3185,7 +3194,7 @@ mod tests {
|
||||||
fn has_active_capture_jobs_false_for_completed() {
|
fn has_active_capture_jobs_false_for_completed() {
|
||||||
let conn = conn();
|
let conn = conn();
|
||||||
let uid = create_capture_job(&conn, "test").unwrap();
|
let uid = create_capture_job(&conn, "test").unwrap();
|
||||||
update_capture_job_status(&conn, &uid, "completed", Some("run_x"), None).unwrap();
|
update_capture_job_status(&conn, &uid, "completed", Some("run_x"), None, None).unwrap();
|
||||||
assert!(!has_active_capture_jobs(&conn).unwrap());
|
assert!(!has_active_capture_jobs(&conn).unwrap());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,15 @@
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use base64::engine::general_purpose::STANDARD as B64;
|
use base64::engine::general_purpose::STANDARD as B64;
|
||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use std::{collections::HashMap, env, io::Read, path::Path, process::Command};
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
env,
|
||||||
|
io::Read,
|
||||||
|
net::TcpListener,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
process::Command,
|
||||||
|
time::{Duration, Instant},
|
||||||
|
};
|
||||||
|
|
||||||
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
|
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
|
||||||
use crate::hash::hash_file;
|
use crate::hash::hash_file;
|
||||||
|
|
@ -17,26 +25,88 @@ pub struct SaveResult {
|
||||||
pub favicon_hash: Option<String>,
|
pub favicon_hash: Option<String>,
|
||||||
/// File extension for the favicon (e.g. `".ico"`, `".png"`), if present.
|
/// File extension for the favicon (e.g. `".ico"`, `".png"`), if present.
|
||||||
pub favicon_ext: Option<String>,
|
pub favicon_ext: Option<String>,
|
||||||
|
/// `true` when `ARCHIVR_UBLOCK=true` (the default) but the extension path
|
||||||
|
/// was missing or invalid. The capture succeeded but ran without ad-blocking.
|
||||||
|
pub ublock_skipped: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Archives `url` as a self-contained HTML snapshot.
|
/// Archives `url` as a self-contained HTML snapshot.
|
||||||
///
|
///
|
||||||
/// Returns `(sha256_hex, title_hint)` on success.
|
/// Env vars:
|
||||||
/// - `sha256_hex`: hash of the saved `.html` file, used as the blob key.
|
|
||||||
/// - `title_hint`: page title extracted from the `<title>` tag, if present.
|
|
||||||
///
|
|
||||||
/// Reads two env vars:
|
|
||||||
/// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`).
|
/// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`).
|
||||||
/// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`).
|
/// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`).
|
||||||
pub fn save(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<SaveResult> {
|
/// - `ARCHIVR_UBLOCK`: enable uBlock Origin Lite extension (default: `"true"`).
|
||||||
|
/// - `ARCHIVR_UBLOCK_EXT`: path to the unpacked uBlock Origin Lite extension directory.
|
||||||
|
/// - `ARCHIVR_CHROME_ARGS`: space-separated extra Chrome flags (e.g. `"--no-sandbox"`).
|
||||||
|
pub fn save(
|
||||||
|
url: &str,
|
||||||
|
store_path: &Path,
|
||||||
|
timestamp: &str,
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
|
) -> Result<SaveResult> {
|
||||||
let single_file =
|
let single_file =
|
||||||
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
|
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
|
||||||
let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string());
|
let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string());
|
||||||
save_with(url, store_path, timestamp, &single_file, &chrome, cookies)
|
let (ublock_ext, ublock_skipped) = resolve_ublock_config();
|
||||||
|
let mut result = save_with(
|
||||||
|
url,
|
||||||
|
store_path,
|
||||||
|
timestamp,
|
||||||
|
&single_file,
|
||||||
|
&chrome,
|
||||||
|
cookies,
|
||||||
|
ublock_ext.as_deref(),
|
||||||
|
)?;
|
||||||
|
result.ublock_skipped = ublock_skipped;
|
||||||
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Inner implementation; takes binary paths explicitly so tests can inject them
|
/// Reads `ARCHIVR_UBLOCK` and `ARCHIVR_UBLOCK_EXT` and returns:
|
||||||
/// without mutating process-global environment variables.
|
///
|
||||||
|
/// - `(Some(path), false)` — uBlock is enabled and the extension dir is valid.
|
||||||
|
/// - `(None, true)` — uBlock is enabled but the extension dir is missing/invalid
|
||||||
|
/// (warns to stderr; caller should surface this to the user).
|
||||||
|
/// - `(None, false)` — uBlock is explicitly disabled (`ARCHIVR_UBLOCK=false/0`).
|
||||||
|
fn resolve_ublock_config() -> (Option<PathBuf>, bool) {
|
||||||
|
let enabled = env::var("ARCHIVR_UBLOCK").unwrap_or_else(|_| "true".to_string());
|
||||||
|
if enabled.eq_ignore_ascii_case("false") || enabled == "0" {
|
||||||
|
return (None, false);
|
||||||
|
}
|
||||||
|
match env::var("ARCHIVR_UBLOCK_EXT").ok().filter(|s| !s.is_empty()) {
|
||||||
|
None => {
|
||||||
|
eprintln!(
|
||||||
|
"warn: uBlock: ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set; \
|
||||||
|
capturing without ad-blocking"
|
||||||
|
);
|
||||||
|
(None, true)
|
||||||
|
}
|
||||||
|
Some(ext_path_str) => {
|
||||||
|
let path = PathBuf::from(&ext_path_str);
|
||||||
|
if path.is_dir() {
|
||||||
|
(Some(path), false)
|
||||||
|
} else {
|
||||||
|
eprintln!(
|
||||||
|
"warn: uBlock: ARCHIVR_UBLOCK_EXT={ext_path_str:?} is not a directory; \
|
||||||
|
capturing without ad-blocking"
|
||||||
|
);
|
||||||
|
(None, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Inner implementation. Takes binary paths and an optional uBlock extension
|
||||||
|
/// directory explicitly so tests can inject them without touching env vars.
|
||||||
|
///
|
||||||
|
/// When `ublock_ext` is `Some(path)` we own Chrome's lifecycle:
|
||||||
|
/// 1. allocate a free TCP port,
|
||||||
|
/// 2. launch Chrome headless with the extension loaded,
|
||||||
|
/// 3. wait for Chrome's DevTools HTTP API to respond,
|
||||||
|
/// 4. run single-file pointing at our Chrome via `--browser-server`,
|
||||||
|
/// 5. kill Chrome after single-file exits.
|
||||||
|
///
|
||||||
|
/// When `ublock_ext` is `None` the original behaviour is preserved:
|
||||||
|
/// single-file launches and manages Chrome internally.
|
||||||
fn save_with(
|
fn save_with(
|
||||||
url: &str,
|
url: &str,
|
||||||
store_path: &Path,
|
store_path: &Path,
|
||||||
|
|
@ -44,17 +114,18 @@ fn save_with(
|
||||||
single_file: &str,
|
single_file: &str,
|
||||||
chrome: &str,
|
chrome: &str,
|
||||||
cookies: &HashMap<String, String>,
|
cookies: &HashMap<String, String>,
|
||||||
|
ublock_ext: Option<&Path>,
|
||||||
) -> Result<SaveResult> {
|
) -> Result<SaveResult> {
|
||||||
let temp_dir = store_path.join("temp").join(timestamp);
|
let temp_dir = store_path.join("temp").join(timestamp);
|
||||||
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
|
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
|
||||||
|
|
||||||
let out_file = temp_dir.join(format!("{timestamp}.html"));
|
let out_file = temp_dir.join(format!("{timestamp}.html"));
|
||||||
|
|
||||||
// Write a user script that strips <script> elements from the live DOM
|
// User script that strips <script> elements from the live DOM just before
|
||||||
// just before SingleFile serializes it. This lets scripts execute during
|
// SingleFile serializes it. Lets scripts execute during capture (so
|
||||||
// capture (so JS-applied CSS is present) without leaving data:-URL ES
|
// JS-applied CSS is present) without leaving data:-URL ES modules in the
|
||||||
// modules in the saved file that would cause "base scheme isn't
|
// saved file that would cause "base scheme isn't hierarchical" errors.
|
||||||
// hierarchical" errors in the viewer. JSON-LD structured data is kept.
|
// JSON-LD structured data is kept.
|
||||||
let user_script_path = temp_dir.join("sf-strip-scripts.js");
|
let user_script_path = temp_dir.join("sf-strip-scripts.js");
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
&user_script_path,
|
&user_script_path,
|
||||||
|
|
@ -65,38 +136,21 @@ fn save_with(
|
||||||
)
|
)
|
||||||
.context("failed to write single-file user script")?;
|
.context("failed to write single-file user script")?;
|
||||||
|
|
||||||
// Chrome's user-data-dir for this capture. Required alongside
|
// Isolated Chrome profile directory; keeps each capture's browser state
|
||||||
// --disable-web-security — newer Chromium silently ignores that flag
|
// separate and cleans up with the rest of the temp dir.
|
||||||
// without a writable user-data-dir. Using a subdirectory of temp_dir
|
|
||||||
// keeps it isolated and it gets cleaned up with the rest of the temp dir.
|
|
||||||
let chrome_data_dir = temp_dir.join("chrome-data");
|
let chrome_data_dir = temp_dir.join("chrome-data");
|
||||||
// Build the browser-args JSON array. Start with the flags always required,
|
|
||||||
// then append any extra flags from ARCHIVR_CHROME_ARGS (space-separated).
|
|
||||||
// Docker containers running as root need "--no-sandbox" here because
|
|
||||||
// Chromium refuses to start as root without it.
|
|
||||||
//
|
|
||||||
// --window-size is set to a realistic desktop viewport so that
|
|
||||||
// --remove-alternative-medias=false and --remove-unused-styles=false
|
|
||||||
// actually preserve responsive @media rules and styles that only match
|
|
||||||
// at normal screen widths (headless Chromium defaults to a small viewport
|
|
||||||
// that would otherwise defeat the preservation flags).
|
|
||||||
let mut chrome_flags = vec![
|
|
||||||
"--disable-web-security".to_string(),
|
|
||||||
format!("--user-data-dir={}", chrome_data_dir.display()),
|
|
||||||
"--window-size=1920,1080".to_string(),
|
|
||||||
];
|
|
||||||
if let Ok(extra) = std::env::var("ARCHIVR_CHROME_ARGS") {
|
|
||||||
chrome_flags.extend(extra.split_whitespace().filter(|s| !s.is_empty()).map(str::to_string));
|
|
||||||
}
|
|
||||||
let quoted: Vec<String> = chrome_flags
|
|
||||||
.iter()
|
|
||||||
.map(|f| format!("\"{}\"", f.replace('\\', "\\\\").replace('"', "\\\"")))
|
|
||||||
.collect();
|
|
||||||
let browser_args = format!("[{}]", quoted.join(","));
|
|
||||||
|
|
||||||
// Write cookie file if cookies are provided.
|
// Extra Chrome flags from the environment (e.g. "--no-sandbox" in Docker).
|
||||||
|
let extra_chrome_args: Vec<String> = env::var("ARCHIVR_CHROME_ARGS")
|
||||||
|
.unwrap_or_default()
|
||||||
|
.split_whitespace()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Write cookie file before running Chrome so it's available immediately.
|
||||||
// Never pass cookie values in process args (ps exposure).
|
// Never pass cookie values in process args (ps exposure).
|
||||||
let cookie_file: Option<std::path::PathBuf> = if !cookies.is_empty() {
|
let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
|
||||||
let cf = temp_dir.join("cookies.txt");
|
let cf = temp_dir.join("cookies.txt");
|
||||||
let domain = domain_from_url(url);
|
let domain = domain_from_url(url);
|
||||||
write_netscape_cookie_file(cookies, &domain, &cf)
|
write_netscape_cookie_file(cookies, &domain, &cf)
|
||||||
|
|
@ -106,52 +160,90 @@ fn save_with(
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut cmd = Command::new(single_file);
|
let sf_output = match ublock_ext {
|
||||||
cmd.arg(url)
|
// ── We own Chrome's lifecycle (uBlock extension mode) ─────────────
|
||||||
.arg(&out_file)
|
Some(ext_path) => {
|
||||||
.arg(format!("--browser-executable-path={chrome}"))
|
let port = allocate_free_port().context("failed to allocate a free TCP port")?;
|
||||||
.arg("--browser-headless")
|
|
||||||
.arg("--browser-wait-until=networkidle2")
|
|
||||||
// Extra delay after networkidle2: Cloudflare Fonts injects @font-face
|
|
||||||
// CSS after HTML parse, so the font hook needs more time to see it.
|
|
||||||
.arg("--browser-wait-delay=2000")
|
|
||||||
// Realistic UA: some origins block headless Chrome's default UA string.
|
|
||||||
.arg("--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36")
|
|
||||||
// Chrome-level flags: disable CORS so fonts from any CDN origin can be
|
|
||||||
// read and inlined (e.g. fonts.gstatic.com without ACAO:*).
|
|
||||||
.arg(format!("--browser-args={browser_args}"))
|
|
||||||
// Preserve all CSS: single-file's defaults strip rules it considers
|
|
||||||
// "unused" (breaks CSS nesting) and remove @media blocks that don't
|
|
||||||
// match the capture viewport (breaks responsive layout).
|
|
||||||
.arg("--remove-unused-styles=false")
|
|
||||||
.arg("--remove-alternative-medias=false")
|
|
||||||
// Allow scripts to run during capture so JS-applied classes exist in
|
|
||||||
// the DOM when CSS is evaluated. The user script above strips <script>
|
|
||||||
// elements before serialization so no broken module imports end up in
|
|
||||||
// the saved file.
|
|
||||||
.arg("--block-scripts=false")
|
|
||||||
.arg(format!("--browser-script={}", user_script_path.display()))
|
|
||||||
// Preserve fonts: defaults strip @font-face rules deemed "unused" or
|
|
||||||
// "alternative" (unicode-range subsets), losing CDN-served fonts.
|
|
||||||
.arg("--remove-unused-fonts=false")
|
|
||||||
.arg("--remove-alternative-fonts=false");
|
|
||||||
if let Some(cf) = &cookie_file {
|
|
||||||
cmd.arg(format!("--browser-cookies-file={}", cf.display()));
|
|
||||||
}
|
|
||||||
let spawn_result = cmd
|
|
||||||
.output()
|
|
||||||
.with_context(|| format!("failed to spawn {single_file} process"));
|
|
||||||
|
|
||||||
// Delete cookie file unconditionally — including on spawn failure —
|
let mut chrome_flags = vec![
|
||||||
// so secrets are never left in store/temp when the capture fails.
|
"--headless=new".to_string(),
|
||||||
|
format!("--remote-debugging-port={port}"),
|
||||||
|
format!("--user-data-dir={}", chrome_data_dir.display()),
|
||||||
|
// Load the extension; disable all others so no unexpected ext loads.
|
||||||
|
format!("--load-extension={}", ext_path.display()),
|
||||||
|
format!("--disable-extensions-except={}", ext_path.display()),
|
||||||
|
// Allow cross-origin font inlining (same reason as standalone mode).
|
||||||
|
"--disable-web-security".to_string(),
|
||||||
|
// Realistic viewport so responsive @media rules are preserved.
|
||||||
|
"--window-size=1920,1080".to_string(),
|
||||||
|
];
|
||||||
|
chrome_flags.extend(extra_chrome_args);
|
||||||
|
|
||||||
|
let mut chrome_child = Command::new(chrome)
|
||||||
|
.args(&chrome_flags)
|
||||||
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::null())
|
||||||
|
.spawn()
|
||||||
|
.with_context(|| format!("failed to spawn Chrome ({chrome})"))?;
|
||||||
|
|
||||||
|
if !wait_for_chrome_ready(port, 10) {
|
||||||
|
let _ = chrome_child.kill();
|
||||||
|
let _ = chrome_child.wait();
|
||||||
|
bail!("Chrome did not become ready on port {port} within 10 s");
|
||||||
|
}
|
||||||
|
|
||||||
|
let out = run_single_file_with_server(
|
||||||
|
url,
|
||||||
|
&out_file,
|
||||||
|
single_file,
|
||||||
|
port,
|
||||||
|
&user_script_path,
|
||||||
|
cookie_file.as_deref(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Always kill Chrome and reap its exit status, even on single-file failure.
|
||||||
|
let _ = chrome_child.kill();
|
||||||
|
let _ = chrome_child.wait();
|
||||||
|
|
||||||
|
out.with_context(|| format!("failed to spawn single-file ({single_file})"))?
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── single-file manages Chrome (original behaviour) ───────────────
|
||||||
|
None => {
|
||||||
|
let mut chrome_flags = vec![
|
||||||
|
"--disable-web-security".to_string(),
|
||||||
|
format!("--user-data-dir={}", chrome_data_dir.display()),
|
||||||
|
"--window-size=1920,1080".to_string(),
|
||||||
|
];
|
||||||
|
chrome_flags.extend(extra_chrome_args);
|
||||||
|
// single-file expects browser-args as a JSON array of strings.
|
||||||
|
let quoted: Vec<String> = chrome_flags
|
||||||
|
.iter()
|
||||||
|
.map(|f| format!("\"{}\"", f.replace('\\', "\\\\").replace('"', "\\\"")))
|
||||||
|
.collect();
|
||||||
|
let browser_args = format!("[{}]", quoted.join(","));
|
||||||
|
|
||||||
|
run_single_file_standalone(
|
||||||
|
url,
|
||||||
|
&out_file,
|
||||||
|
single_file,
|
||||||
|
chrome,
|
||||||
|
&browser_args,
|
||||||
|
&user_script_path,
|
||||||
|
cookie_file.as_deref(),
|
||||||
|
)
|
||||||
|
.with_context(|| format!("failed to spawn single-file ({single_file})"))?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Delete cookie file unconditionally — including on failure — so secrets
|
||||||
|
// are never left in store/temp when the capture fails.
|
||||||
if let Some(cf) = &cookie_file {
|
if let Some(cf) = &cookie_file {
|
||||||
let _ = std::fs::remove_file(cf);
|
let _ = std::fs::remove_file(cf);
|
||||||
}
|
}
|
||||||
|
|
||||||
let out = spawn_result?;
|
if !sf_output.status.success() {
|
||||||
|
let stderr = String::from_utf8_lossy(&sf_output.stderr);
|
||||||
if !out.status.success() {
|
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
|
||||||
bail!("single-file failed: {stderr}");
|
bail!("single-file failed: {stderr}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -164,12 +256,127 @@ fn save_with(
|
||||||
|
|
||||||
let title = extract_html_title(&out_file);
|
let title = extract_html_title(&out_file);
|
||||||
let html_hash = hash_file(&out_file)?;
|
let html_hash = hash_file(&out_file)?;
|
||||||
let (favicon_hash, favicon_ext) = extract_and_save_favicon(&out_file, &temp_dir, timestamp)
|
let (favicon_hash, favicon_ext) =
|
||||||
.map(|(h, e)| (Some(h), Some(e)))
|
extract_and_save_favicon(&out_file, &temp_dir, timestamp)
|
||||||
.unwrap_or((None, None));
|
.map(|(h, e)| (Some(h), Some(e)))
|
||||||
Ok(SaveResult { html_hash, title, favicon_hash, favicon_ext })
|
.unwrap_or((None, None));
|
||||||
|
|
||||||
|
Ok(SaveResult {
|
||||||
|
html_hash,
|
||||||
|
title,
|
||||||
|
favicon_hash,
|
||||||
|
favicon_ext,
|
||||||
|
ublock_skipped: false, // overwritten by save() from resolve_ublock_config()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Chrome helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Binds a `TcpListener` to a random OS-assigned port, reads the port number,
|
||||||
|
/// then drops the listener. The tiny TOCTOU window between drop and Chrome
|
||||||
|
/// binding is acceptable in practice.
|
||||||
|
fn allocate_free_port() -> Result<u16> {
|
||||||
|
let listener =
|
||||||
|
TcpListener::bind("127.0.0.1:0").context("could not bind to a free TCP port")?;
|
||||||
|
Ok(listener.local_addr()?.port())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Polls `http://127.0.0.1:{port}/json/version` every 150 ms until Chrome
|
||||||
|
/// responds with HTTP 200 or the deadline (timeout_secs) elapses.
|
||||||
|
fn wait_for_chrome_ready(port: u16, timeout_secs: u64) -> bool {
|
||||||
|
let url = format!("http://127.0.0.1:{port}/json/version");
|
||||||
|
let client = match reqwest::blocking::Client::builder()
|
||||||
|
.timeout(Duration::from_millis(500))
|
||||||
|
.build()
|
||||||
|
{
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(_) => return false,
|
||||||
|
};
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
if client
|
||||||
|
.get(&url)
|
||||||
|
.send()
|
||||||
|
.map(|r| r.status().is_success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(150));
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── single-file invocation helpers ───────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Runs single-file pointing at an already-running Chrome via the DevTools HTTP
|
||||||
|
/// API (`--browser-server`). Chrome was started by the caller, which retains
|
||||||
|
/// ownership of the process handle and kills it after this call returns.
|
||||||
|
fn run_single_file_with_server(
|
||||||
|
url: &str,
|
||||||
|
out_file: &Path,
|
||||||
|
single_file: &str,
|
||||||
|
port: u16,
|
||||||
|
user_script_path: &Path,
|
||||||
|
cookie_file: Option<&Path>,
|
||||||
|
) -> std::io::Result<std::process::Output> {
|
||||||
|
let mut cmd = base_single_file_cmd(url, out_file, single_file, user_script_path, cookie_file);
|
||||||
|
cmd.arg(format!("--browser-server=http://127.0.0.1:{port}"));
|
||||||
|
cmd.output()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs single-file, letting it launch and manage Chrome itself.
|
||||||
|
fn run_single_file_standalone(
|
||||||
|
url: &str,
|
||||||
|
out_file: &Path,
|
||||||
|
single_file: &str,
|
||||||
|
chrome: &str,
|
||||||
|
browser_args: &str, // JSON array string, e.g. `["--disable-web-security",...]`
|
||||||
|
user_script_path: &Path,
|
||||||
|
cookie_file: Option<&Path>,
|
||||||
|
) -> std::io::Result<std::process::Output> {
|
||||||
|
let mut cmd = base_single_file_cmd(url, out_file, single_file, user_script_path, cookie_file);
|
||||||
|
cmd.arg(format!("--browser-executable-path={chrome}"))
|
||||||
|
.arg("--browser-headless")
|
||||||
|
.arg(format!("--browser-args={browser_args}"));
|
||||||
|
cmd.output()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds a `Command` with the single-file args that are the same regardless
|
||||||
|
/// of how Chrome is started.
|
||||||
|
fn base_single_file_cmd(
|
||||||
|
url: &str,
|
||||||
|
out_file: &Path,
|
||||||
|
single_file: &str,
|
||||||
|
user_script_path: &Path,
|
||||||
|
cookie_file: Option<&Path>,
|
||||||
|
) -> Command {
|
||||||
|
let mut cmd = Command::new(single_file);
|
||||||
|
cmd.arg(url)
|
||||||
|
.arg(out_file)
|
||||||
|
.arg("--browser-wait-until=networkidle2")
|
||||||
|
// Extra delay after networkidle2: Cloudflare Fonts injects @font-face
|
||||||
|
// CSS after HTML parse, so the font hook needs more time to see it.
|
||||||
|
.arg("--browser-wait-delay=2000")
|
||||||
|
// Realistic UA: some origins block headless Chrome's default UA string.
|
||||||
|
.arg("--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36")
|
||||||
|
// Preserve all CSS.
|
||||||
|
.arg("--remove-unused-styles=false")
|
||||||
|
.arg("--remove-alternative-medias=false")
|
||||||
|
// Allow scripts during capture so JS-applied classes exist in the DOM.
|
||||||
|
.arg("--block-scripts=false")
|
||||||
|
.arg(format!("--browser-script={}", user_script_path.display()))
|
||||||
|
// Preserve fonts.
|
||||||
|
.arg("--remove-unused-fonts=false")
|
||||||
|
.arg("--remove-alternative-fonts=false");
|
||||||
|
if let Some(cf) = cookie_file {
|
||||||
|
cmd.arg(format!("--browser-cookies-file={}", cf.display()));
|
||||||
|
}
|
||||||
|
cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── HTML helpers ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Reads the first 8 KiB of `path` and extracts the content of the first
|
/// Reads the first 8 KiB of `path` and extracts the content of the first
|
||||||
/// `<title>…</title>` element. Returns `None` if absent or empty.
|
/// `<title>…</title>` element. Returns `None` if absent or empty.
|
||||||
///
|
///
|
||||||
|
|
@ -177,21 +384,17 @@ fn save_with(
|
||||||
/// lowercasing is byte-length-preserving, so byte offsets derived from the
|
/// lowercasing is byte-length-preserving, so byte offsets derived from the
|
||||||
/// lowercased buffer are valid indices into the original buffer.
|
/// lowercased buffer are valid indices into the original buffer.
|
||||||
fn extract_html_title(path: &Path) -> Option<String> {
|
fn extract_html_title(path: &Path) -> Option<String> {
|
||||||
let mut buf = [0u8; 8192];
|
let mut f = std::fs::File::open(path).ok()?;
|
||||||
let n = std::fs::File::open(path).ok()?.read(&mut buf).ok()?;
|
let mut buf = vec![0u8; 8192];
|
||||||
// Recover a valid UTF-8 prefix if the 8 KiB boundary falls mid-character.
|
let n = f.read(&mut buf).ok()?;
|
||||||
let snippet = match std::str::from_utf8(&buf[..n]) {
|
let buf = &buf[..n];
|
||||||
Ok(s) => s,
|
let lower = String::from_utf8_lossy(buf).to_ascii_lowercase();
|
||||||
Err(e) => std::str::from_utf8(&buf[..e.valid_up_to()]).ok()?,
|
let start = lower.find("<title>")? + "<title>".len();
|
||||||
};
|
let end = lower[start..].find("</title>")? + start;
|
||||||
// ASCII-only lowercase: A-Z -> a-z, all other bytes unchanged.
|
let title = String::from_utf8_lossy(&buf[start..end])
|
||||||
// Byte lengths are identical to the original, so offsets are safe to reuse.
|
.trim()
|
||||||
let lower = snippet.to_ascii_lowercase();
|
.to_string();
|
||||||
let tag_start = lower.find("<title>")?;
|
if title.is_empty() { None } else { Some(title) }
|
||||||
let content_start = tag_start + 7; // len("<title>") == 7
|
|
||||||
let content_end = content_start + lower[content_start..].find("</title>")?;
|
|
||||||
let title = snippet[content_start..content_end].trim();
|
|
||||||
if title.is_empty() { None } else { Some(title.to_string()) }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extracts the favicon embedded in a single-file HTML archive.
|
/// Extracts the favicon embedded in a single-file HTML archive.
|
||||||
|
|
@ -205,71 +408,54 @@ fn extract_and_save_favicon(
|
||||||
temp_dir: &Path,
|
temp_dir: &Path,
|
||||||
timestamp: &str,
|
timestamp: &str,
|
||||||
) -> Option<(String, String)> {
|
) -> Option<(String, String)> {
|
||||||
let html = std::fs::read_to_string(html_path).ok()?;
|
let content = std::fs::read_to_string(html_path).ok()?;
|
||||||
let lower = html.to_ascii_lowercase();
|
let lower = content.to_ascii_lowercase();
|
||||||
|
|
||||||
// Find a <link> tag that has rel="...icon..." AND href="data:image/..."
|
// Find the first <link …> tag that looks like a favicon with a data: href.
|
||||||
let link_start = {
|
let mut search_pos = 0;
|
||||||
let mut found = None;
|
loop {
|
||||||
let mut search = 0;
|
let tag_start = lower[search_pos..].find("<link")? + search_pos;
|
||||||
while search < lower.len() {
|
let tag_end = lower[tag_start..].find('>')? + tag_start;
|
||||||
let off = lower[search..].find("<link")?;
|
let tag = &lower[tag_start..=tag_end];
|
||||||
let abs = search + off;
|
|
||||||
// Find end of this tag, respecting quoted attribute values so that
|
if tag.contains("icon") {
|
||||||
// a '>' inside a data URL does not terminate the tag prematurely.
|
// Look for href="data:image/...;base64,..."
|
||||||
let tag_slice = &lower[abs..];
|
if let Some(href_pos) = tag.find("href=") {
|
||||||
let mut in_q = false;
|
let after_href = &content[tag_start + href_pos + 5..];
|
||||||
let mut tag_end = None;
|
let (quote, after_quote) = if after_href.starts_with('"') {
|
||||||
for (i, c) in tag_slice.char_indices() {
|
('"', &after_href[1..])
|
||||||
match c {
|
} else if after_href.starts_with('\'') {
|
||||||
'"' => in_q = !in_q,
|
('\'', &after_href[1..])
|
||||||
'>' if !in_q => { tag_end = Some(i); break; }
|
} else {
|
||||||
_ => {}
|
search_pos = tag_end + 1;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let value_end = after_quote.find(quote)?;
|
||||||
|
let href_value = &after_quote[..value_end];
|
||||||
|
if let Some(b64_start) = href_value.to_ascii_lowercase().find(";base64,") {
|
||||||
|
let mime_part = &href_value[5..b64_start]; // skip "data:"
|
||||||
|
let ext = mime_to_favicon_ext(mime_part)?;
|
||||||
|
let b64_data = &href_value[b64_start + 8..];
|
||||||
|
let bytes = B64.decode(b64_data).ok()?;
|
||||||
|
let out_path = temp_dir.join(format!("{timestamp}.favicon{ext}"));
|
||||||
|
std::fs::write(&out_path, &bytes).ok()?;
|
||||||
|
let hash = hash_file(&out_path).ok()?;
|
||||||
|
return Some((hash, ext.to_string()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let tag_end = match tag_end { Some(e) => e, None => break };
|
|
||||||
let tag_s = &lower[abs..abs + tag_end];
|
|
||||||
if tag_s.contains("rel=") && tag_s.contains("icon") && tag_s.contains("href=\"data:image") {
|
|
||||||
found = Some(abs);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
search = abs + tag_end + 1;
|
|
||||||
}
|
}
|
||||||
found?
|
|
||||||
};
|
|
||||||
|
|
||||||
// Extract href value from the original HTML (byte positions match because
|
search_pos = tag_end + 1;
|
||||||
// to_ascii_lowercase is byte-length-preserving).
|
|
||||||
let tag_lower = &lower[link_start..];
|
|
||||||
let href_off = tag_lower.find("href=\"")?;
|
|
||||||
let value_start = link_start + href_off + 6; // past href="
|
|
||||||
let value_end = html[value_start..].find('"')?;
|
|
||||||
let data_url = &html[value_start..value_start + value_end];
|
|
||||||
|
|
||||||
// Parse data:<mime>;base64,<payload>
|
|
||||||
let rest = data_url.strip_prefix("data:")?;
|
|
||||||
let comma = rest.find(',')?;
|
|
||||||
let meta = &rest[..comma];
|
|
||||||
let b64 = &rest[comma + 1..];
|
|
||||||
if !meta.to_ascii_lowercase().contains("base64") {
|
|
||||||
return None;
|
|
||||||
}
|
}
|
||||||
let mime = meta.split(';').next()?.trim().to_ascii_lowercase();
|
|
||||||
let ext = mime_to_favicon_ext(&mime)?;
|
|
||||||
|
|
||||||
let bytes = B64.decode(b64.trim()).ok()?;
|
|
||||||
let out = temp_dir.join(format!("{timestamp}.favicon{ext}"));
|
|
||||||
std::fs::write(&out, &bytes).ok()?;
|
|
||||||
hash_file(&out).ok().map(|h| (h, ext.to_string()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mime_to_favicon_ext(mime: &str) -> Option<&'static str> {
|
fn mime_to_favicon_ext(mime: &str) -> Option<&'static str> {
|
||||||
match mime {
|
match mime.to_ascii_lowercase().trim() {
|
||||||
"image/x-icon" | "image/vnd.microsoft.icon" => Some(".ico"),
|
"image/x-icon" | "image/vnd.microsoft.icon" => Some(".ico"),
|
||||||
"image/png" => Some(".png"),
|
"image/png" => Some(".png"),
|
||||||
"image/svg+xml" => Some(".svg"),
|
|
||||||
"image/jpeg" => Some(".jpg"),
|
"image/jpeg" => Some(".jpg"),
|
||||||
"image/gif" => Some(".gif"),
|
"image/gif" => Some(".gif"),
|
||||||
|
"image/svg+xml" => Some(".svg"),
|
||||||
"image/webp" => Some(".webp"),
|
"image/webp" => Some(".webp"),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
|
|
@ -323,6 +509,7 @@ mod tests {
|
||||||
"/nonexistent/single-file",
|
"/nonexistent/single-file",
|
||||||
"chromium",
|
"chromium",
|
||||||
&HashMap::new(),
|
&HashMap::new(),
|
||||||
|
None, // no ublock ext
|
||||||
);
|
);
|
||||||
let err = result.unwrap_err();
|
let err = result.unwrap_err();
|
||||||
let msg = format!("{err:#}");
|
let msg = format!("{err:#}");
|
||||||
|
|
@ -331,4 +518,19 @@ mod tests {
|
||||||
"unexpected error: {msg}"
|
"unexpected error: {msg}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_ublock_config_disabled_when_false() {
|
||||||
|
// Can't mutate env vars safely in parallel tests; test the logic directly
|
||||||
|
// by verifying the env-var parsing branch we care about.
|
||||||
|
let enabled = "false";
|
||||||
|
let is_disabled =
|
||||||
|
enabled.eq_ignore_ascii_case("false") || enabled == "0";
|
||||||
|
assert!(is_disabled);
|
||||||
|
|
||||||
|
let enabled = "0";
|
||||||
|
let is_disabled =
|
||||||
|
enabled.eq_ignore_ascii_case("false") || enabled == "0";
|
||||||
|
assert!(is_disabled);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -756,15 +756,21 @@ async fn capture_handler(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok();
|
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
|
||||||
match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref(), &capture_config) {
|
match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref(), &capture_config) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
|
let notes = if result.ublock_skipped {
|
||||||
|
Some(r#"{"ublock_skipped":true}"#)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
database::update_capture_job_status(
|
database::update_capture_job_status(
|
||||||
&conn,
|
&conn,
|
||||||
&job_uid_bg,
|
&job_uid_bg,
|
||||||
"completed",
|
"completed",
|
||||||
Some(&result.run_uid),
|
Some(&result.run_uid),
|
||||||
None,
|
None,
|
||||||
|
notes,
|
||||||
)
|
)
|
||||||
.ok();
|
.ok();
|
||||||
}
|
}
|
||||||
|
|
@ -775,6 +781,7 @@ async fn capture_handler(
|
||||||
"failed",
|
"failed",
|
||||||
None,
|
None,
|
||||||
Some(&format!("{e:#}")),
|
Some(&format!("{e:#}")),
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.ok();
|
.ok();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
40
crates/archivr-server/static/assets/index-BpXoSc97.js
Normal file
40
crates/archivr-server/static/assets/index-BpXoSc97.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>Archivr</title>
|
<title>Archivr</title>
|
||||||
<script type="module" crossorigin src="/assets/index-CHDuICqH.js"></script>
|
<script type="module" crossorigin src="/assets/index-BpXoSc97.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-BGSI2WEx.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div id="root"></div>
|
<div id="root"></div>
|
||||||
|
|
|
||||||
20
flake.nix
20
flake.nix
|
|
@ -60,6 +60,22 @@
|
||||||
tweetPython = pkgs.python312.withPackages (ps: [
|
tweetPython = pkgs.python312.withPackages (ps: [
|
||||||
twitterApiClient
|
twitterApiClient
|
||||||
]);
|
]);
|
||||||
|
# uBlock Origin Lite (MV3) — unpacked Chromium extension for headless ad-blocking.
|
||||||
|
# Fetched from the uBOL-home GitHub releases; update version + hash together.
|
||||||
|
ublockLite = pkgs.stdenv.mkDerivation {
|
||||||
|
pname = "ublock-origin-lite";
|
||||||
|
version = "2026.705.2152";
|
||||||
|
src = pkgs.fetchurl {
|
||||||
|
url = "https://github.com/uBlockOrigin/uBOL-home/releases/download/2026.705.2152/uBOLite_2026.705.2152.chromium.zip";
|
||||||
|
hash = "sha256-4TbvDYbkOkDuVK17TeAbLDBcgf9O6f/vh2buGbLu4XQ=";
|
||||||
|
};
|
||||||
|
nativeBuildInputs = [ pkgs.unzip ];
|
||||||
|
sourceRoot = ".";
|
||||||
|
installPhase = ''
|
||||||
|
mkdir -p $out
|
||||||
|
cp -r . $out/
|
||||||
|
'';
|
||||||
|
};
|
||||||
version = "0.1.0";
|
version = "0.1.0";
|
||||||
src = pkgs.lib.cleanSource ./.;
|
src = pkgs.lib.cleanSource ./.;
|
||||||
cargoLock = {
|
cargoLock = {
|
||||||
|
|
@ -123,6 +139,7 @@
|
||||||
${lib.optionalString pkgs.stdenv.isLinux "--set ARCHIVR_CHROME ${pkgs.chromium}/bin/chromium"} \
|
${lib.optionalString pkgs.stdenv.isLinux "--set ARCHIVR_CHROME ${pkgs.chromium}/bin/chromium"} \
|
||||||
--set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \
|
--set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \
|
||||||
--set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr/scrape_user_tweet_contents.py \
|
--set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr/scrape_user_tweet_contents.py \
|
||||||
|
--set ARCHIVR_UBLOCK_EXT ${ublockLite} \
|
||||||
--prefix PATH : ${
|
--prefix PATH : ${
|
||||||
lib.makeBinPath ([
|
lib.makeBinPath ([
|
||||||
pkgs.yt-dlp
|
pkgs.yt-dlp
|
||||||
|
|
@ -149,7 +166,8 @@
|
||||||
--set ARCHIVR_SINGLE_FILE ${pkgs.single-file-cli}/bin/single-file \
|
--set ARCHIVR_SINGLE_FILE ${pkgs.single-file-cli}/bin/single-file \
|
||||||
${lib.optionalString pkgs.stdenv.isLinux "--set ARCHIVR_CHROME ${pkgs.chromium}/bin/chromium"} \
|
${lib.optionalString pkgs.stdenv.isLinux "--set ARCHIVR_CHROME ${pkgs.chromium}/bin/chromium"} \
|
||||||
--set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \
|
--set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \
|
||||||
--set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr-server/scrape_user_tweet_contents.py
|
--set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr-server/scrape_user_tweet_contents.py \
|
||||||
|
--set ARCHIVR_UBLOCK_EXT ${ublockLite}
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
archivr-all = pkgs.symlinkJoin {
|
archivr-all = pkgs.symlinkJoin {
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,9 @@ export default function App() {
|
||||||
|
|
||||||
const [toasts, setToasts] = useState([])
|
const [toasts, setToasts] = useState([])
|
||||||
const toastIdRef = useRef(0)
|
const toastIdRef = useRef(0)
|
||||||
|
const [ublockWarningIgnored, setUblockWarningIgnored] = useState(
|
||||||
|
() => sessionStorage.getItem('ublockWarningIgnored') === 'true'
|
||||||
|
)
|
||||||
|
|
||||||
const humanizeTags = currentUser?.humanize_slugs ?? false;
|
const humanizeTags = currentUser?.humanize_slugs ?? false;
|
||||||
|
|
||||||
|
|
@ -238,15 +241,22 @@ export default function App() {
|
||||||
])
|
])
|
||||||
}, [archiveId, searchQuery, tagFilter, loadEntries])
|
}, [archiveId, searchQuery, tagFilter, loadEntries])
|
||||||
|
|
||||||
const handleToast = useCallback((errorText, locator) => {
|
const handleToast = useCallback((text, locator, type = 'error') => {
|
||||||
|
if (type === 'warning' && ublockWarningIgnored) return
|
||||||
const id = ++toastIdRef.current
|
const id = ++toastIdRef.current
|
||||||
setToasts(prev => [...prev, { id, errorText, locator }])
|
setToasts(prev => [...prev, { id, text, locator, type }])
|
||||||
}, [])
|
}, [ublockWarningIgnored])
|
||||||
|
|
||||||
const handleDismissToast = useCallback((id) => {
|
const handleDismissToast = useCallback((id) => {
|
||||||
setToasts(prev => prev.filter(t => t.id !== id))
|
setToasts(prev => prev.filter(t => t.id !== id))
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
|
const handleIgnoreUblock = useCallback(() => {
|
||||||
|
sessionStorage.setItem('ublockWarningIgnored', 'true')
|
||||||
|
setUblockWarningIgnored(true)
|
||||||
|
setToasts(prev => prev.filter(t => t.type !== 'warning'))
|
||||||
|
}, [])
|
||||||
|
|
||||||
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;
|
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;
|
||||||
if (authState === 'setup') return <SetupPage onComplete={() => setAuthState('login')} />;
|
if (authState === 'setup') return <SetupPage onComplete={() => setAuthState('login')} />;
|
||||||
if (authState === 'login') return <LoginPage onLogin={user => { setCurrentUser(user); setAuthState('authenticated'); }} />;
|
if (authState === 'login') return <LoginPage onLogin={user => { setCurrentUser(user); setAuthState('authenticated'); }} />;
|
||||||
|
|
@ -347,7 +357,7 @@ export default function App() {
|
||||||
onCaptured={handleCaptured}
|
onCaptured={handleCaptured}
|
||||||
onToast={handleToast}
|
onToast={handleToast}
|
||||||
/>
|
/>
|
||||||
<ToastStack toasts={toasts} onDismiss={handleDismissToast} />
|
<ToastStack toasts={toasts} onDismiss={handleDismissToast} onIgnoreUblock={handleIgnoreUblock} />
|
||||||
</>
|
</>
|
||||||
</AuthContext.Provider>
|
</AuthContext.Provider>
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -186,6 +186,13 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
})
|
})
|
||||||
}, 1400)
|
}, 1400)
|
||||||
onCapturedRef.current()
|
onCapturedRef.current()
|
||||||
|
// Warn if uBlock was requested but the extension wasn't available
|
||||||
|
try {
|
||||||
|
const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null
|
||||||
|
if (notes?.ublock_skipped) {
|
||||||
|
onToastRef.current(null, locator, 'warning')
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
} else if (updated.status === 'failed') {
|
} else if (updated.status === 'failed') {
|
||||||
clearInterval(pollIntervals.current.get(jobUid))
|
clearInterval(pollIntervals.current.get(jobUid))
|
||||||
pollIntervals.current.delete(jobUid)
|
pollIntervals.current.delete(jobUid)
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,23 @@
|
||||||
import { useState, useEffect } from 'react'
|
import { useState, useEffect } from 'react'
|
||||||
|
|
||||||
const TOAST_TTL = 7000 // ms before auto-dismiss; paused when error is expanded
|
const TOAST_TTL = 7000 // ms before auto-dismiss; paused when detail is expanded
|
||||||
|
|
||||||
export default function ToastStack({ toasts, onDismiss }) {
|
export default function ToastStack({ toasts, onDismiss, onIgnoreUblock }) {
|
||||||
if (!toasts.length) return null
|
if (!toasts.length) return null
|
||||||
return (
|
return (
|
||||||
<div className="toast-stack" role="log" aria-live="polite" aria-label="Notifications">
|
<div className="toast-stack" role="log" aria-live="polite" aria-label="Notifications">
|
||||||
{toasts.map(t => (
|
{toasts.map(t => (
|
||||||
<Toast key={t.id} toast={t} onDismiss={onDismiss} />
|
<Toast key={t.id} toast={t} onDismiss={onDismiss} onIgnoreUblock={onIgnoreUblock} />
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
function Toast({ toast, onDismiss }) {
|
function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||||
const [expanded, setExpanded] = useState(false)
|
const [expanded, setExpanded] = useState(false)
|
||||||
|
const isWarning = toast.type === 'warning'
|
||||||
|
|
||||||
// Auto-dismiss after TTL; paused while error detail is expanded
|
// Auto-dismiss after TTL; paused while detail is expanded
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (expanded) return
|
if (expanded) return
|
||||||
const timer = setTimeout(() => onDismiss(toast.id), TOAST_TTL)
|
const timer = setTimeout(() => onDismiss(toast.id), TOAST_TTL)
|
||||||
|
|
@ -27,6 +28,50 @@ function Toast({ toast, onDismiss }) {
|
||||||
? (toast.locator.length > 48 ? toast.locator.slice(0, 45) + '\u2026' : toast.locator)
|
? (toast.locator.length > 48 ? toast.locator.slice(0, 45) + '\u2026' : toast.locator)
|
||||||
: null
|
: null
|
||||||
|
|
||||||
|
if (isWarning) {
|
||||||
|
return (
|
||||||
|
<div className="toast toast--warning" role="alert" aria-atomic="true">
|
||||||
|
<div className="toast-top">
|
||||||
|
<span className="toast-icon" aria-hidden="true">⚠</span>
|
||||||
|
<div className="toast-body">
|
||||||
|
<span className="toast-headline">Ad-blocking unavailable</span>
|
||||||
|
{short && <span className="toast-locator">{short}</span>}
|
||||||
|
</div>
|
||||||
|
<div className="toast-btns">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="toast-view-btn"
|
||||||
|
onClick={() => setExpanded(v => !v)}
|
||||||
|
aria-expanded={expanded}
|
||||||
|
>
|
||||||
|
{expanded ? 'Hide' : 'Details'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="toast-view-btn toast-ignore-btn"
|
||||||
|
onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }}
|
||||||
|
>
|
||||||
|
Ignore
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="toast-dismiss"
|
||||||
|
onClick={() => onDismiss(toast.id)}
|
||||||
|
aria-label="Dismiss"
|
||||||
|
>
|
||||||
|
×
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{expanded && (
|
||||||
|
<p className="toast-warning-detail">
|
||||||
|
{toast.text || 'ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid. The page was captured without ad-blocking. Set ARCHIVR_UBLOCK_EXT to the unpacked uBlock Origin Lite extension directory to enable ad-blocking, or set ARCHIVR_UBLOCK=false to silence this warning.'}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="toast toast--error" role="alert" aria-atomic="true">
|
<div className="toast toast--error" role="alert" aria-atomic="true">
|
||||||
<div className="toast-top">
|
<div className="toast-top">
|
||||||
|
|
@ -36,7 +81,7 @@ function Toast({ toast, onDismiss }) {
|
||||||
{short && <span className="toast-locator">{short}</span>}
|
{short && <span className="toast-locator">{short}</span>}
|
||||||
</div>
|
</div>
|
||||||
<div className="toast-btns">
|
<div className="toast-btns">
|
||||||
{toast.errorText && (
|
{toast.text && (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
className="toast-view-btn"
|
className="toast-view-btn"
|
||||||
|
|
@ -55,8 +100,8 @@ function Toast({ toast, onDismiss }) {
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{expanded && toast.errorText && (
|
{expanded && toast.text && (
|
||||||
<pre className="toast-error-detail">{toast.errorText}</pre>
|
<pre className="toast-error-detail">{toast.text}</pre>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -812,6 +812,7 @@ select {
|
||||||
to { opacity: 1; transform: translateY(0) scale(1); }
|
to { opacity: 1; transform: translateY(0) scale(1); }
|
||||||
}
|
}
|
||||||
.toast--error { border-left: 3px solid var(--accent); }
|
.toast--error { border-left: 3px solid var(--accent); }
|
||||||
|
.toast--warning { border-left: 3px solid #e8a000; }
|
||||||
|
|
||||||
.toast-top {
|
.toast-top {
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|
@ -892,6 +893,18 @@ select {
|
||||||
max-height: 200px;
|
max-height: 200px;
|
||||||
overflow-y: auto;
|
overflow-y: auto;
|
||||||
}
|
}
|
||||||
|
.toast-warning-detail {
|
||||||
|
margin: 0;
|
||||||
|
padding: 10px 14px 12px;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.55;
|
||||||
|
color: var(--muted);
|
||||||
|
background: var(--paper-2);
|
||||||
|
border-top: 1px solid var(--line);
|
||||||
|
white-space: pre-wrap;
|
||||||
|
word-break: break-word;
|
||||||
|
}
|
||||||
|
.toast-ignore-btn { color: var(--muted); }
|
||||||
|
|
||||||
/* ── Utility ─────────────────────────────────────────────────────────────── */
|
/* ── Utility ─────────────────────────────────────────────────────────────── */
|
||||||
.muted { color: var(--muted); }
|
.muted { color: var(--muted); }
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue