From ee697625fba52fd9529ab79c1742e235f7cebb63 Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Wed, 24 Jun 2026 19:16:21 +0200 Subject: [PATCH] feat(singlefile): add SaveResult, favicon extraction, wait for networkidle2 --- Cargo.lock | 1 + Cargo.toml | 1 + crates/archivr-core/Cargo.toml | 1 + .../archivr-core/src/downloader/singlefile.rs | 108 +++++++++++++++++- 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8f45f1..9ce1bd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,6 +106,7 @@ name = "archivr-core" version = "0.1.0" dependencies = [ "anyhow", + "base64", "chrono", "hex", "regex", diff --git a/Cargo.toml b/Cargo.toml index 5a9d811..f88610d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,3 +28,4 @@ tower = "0.5.1" tower-http = { version = "0.6.2", features = ["fs", "trace"] } uuid = { version = "1.18.1", features = ["v4"] } reqwest = { version = "0.12", features = ["blocking"] } +base64 = "0.22" diff --git a/crates/archivr-core/Cargo.toml b/crates/archivr-core/Cargo.toml index 30ecc0f..8192c98 100644 --- a/crates/archivr-core/Cargo.toml +++ b/crates/archivr-core/Cargo.toml @@ -14,6 +14,7 @@ serde_json.workspace = true sha3.workspace = true uuid.workspace = true reqwest = { workspace = true } +base64.workspace = true [dev-dependencies] tempfile = "3" diff --git a/crates/archivr-core/src/downloader/singlefile.rs b/crates/archivr-core/src/downloader/singlefile.rs index 06b409a..c31328f 100644 --- a/crates/archivr-core/src/downloader/singlefile.rs +++ b/crates/archivr-core/src/downloader/singlefile.rs @@ -1,8 +1,23 @@ use anyhow::{Context, Result, bail}; +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine as _; use std::{env, io::Read, path::Path, process::Command}; use crate::hash::hash_file; +/// Result of archiving a web page with single-file. +#[derive(Debug)] +pub struct SaveResult { + /// SHA-256 hex of the archived `.html` file. + pub html_hash: String, + /// Page title from `` tag, if present. + pub title: Option<String>, + /// SHA-256 hex of the extracted favicon, if present. + pub favicon_hash: Option<String>, + /// File extension for the favicon (e.g. `".ico"`, `".png"`), if present. + pub favicon_ext: Option<String>, +} + /// Archives `url` as a self-contained HTML snapshot. /// /// Returns `(sha256_hex, title_hint)` on success. @@ -12,7 +27,7 @@ use crate::hash::hash_file; /// Reads two env vars: /// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`). /// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`). -pub fn save(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, Option<String>)> { +pub fn save(url: &str, store_path: &Path, timestamp: &str) -> Result<SaveResult> { let single_file = env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string()); let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string()); @@ -27,7 +42,7 @@ fn save_with( timestamp: &str, single_file: &str, chrome: &str, -) -> Result<(String, Option<String>)> { +) -> Result<SaveResult> { let temp_dir = store_path.join("temp").join(timestamp); std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?; @@ -38,6 +53,7 @@ fn save_with( .arg(&out_file) .arg(format!("--browser-executable-path={chrome}")) .arg("--browser-headless") + .arg("--browser-wait-until=networkidle2") .output() .with_context(|| format!("failed to spawn {single_file} process"))?; @@ -54,8 +70,11 @@ fn save_with( } let title = extract_html_title(&out_file); - let hash = hash_file(&out_file)?; - Ok((hash, title)) + let html_hash = hash_file(&out_file)?; + let (favicon_hash, favicon_ext) = extract_and_save_favicon(&out_file, &temp_dir, timestamp) + .map(|(h, e)| (Some(h), Some(e))) + .unwrap_or((None, None)); + Ok(SaveResult { html_hash, title, favicon_hash, favicon_ext }) } /// Reads the first 8 KiB of `path` and extracts the content of the first @@ -82,6 +101,87 @@ fn extract_html_title(path: &Path) -> Option<String> { if title.is_empty() { None } else { Some(title.to_string()) } } +/// Extracts the favicon embedded in a single-file HTML archive. +/// +/// Scans for a `<link rel="…icon…">` tag whose `href` is a `data:image/…;base64,…` URL. +/// Decodes the base64 payload, writes it to `{temp_dir}/{timestamp}.favicon.{ext}`, +/// hashes the file, and returns `(sha256_hex, ".ext")`. +/// All failures are silent (returns `None`) — a missing favicon is non-fatal. +fn extract_and_save_favicon( + html_path: &Path, + temp_dir: &Path, + timestamp: &str, +) -> Option<(String, String)> { + let html = std::fs::read_to_string(html_path).ok()?; + let lower = html.to_ascii_lowercase(); + + // Find a <link> tag that has rel="...icon..." AND href="data:image/..." + let link_start = { + let mut found = None; + let mut search = 0; + while search < lower.len() { + let off = lower[search..].find("<link")?; + let abs = search + off; + // Find end of this tag, respecting quoted attribute values so that + // a '>' inside a data URL does not terminate the tag prematurely. + let tag_slice = &lower[abs..]; + let mut in_q = false; + let mut tag_end = None; + for (i, c) in tag_slice.char_indices() { + match c { + '"' => in_q = !in_q, + '>' if !in_q => { tag_end = Some(i); break; } + _ => {} + } + } + let tag_end = match tag_end { Some(e) => e, None => break }; + let tag_s = &lower[abs..abs + tag_end]; + if tag_s.contains("rel=") && tag_s.contains("icon") && tag_s.contains("href=\"data:image") { + found = Some(abs); + break; + } + search = abs + tag_end + 1; + } + found? + }; + + // Extract href value from the original HTML (byte positions match because + // to_ascii_lowercase is byte-length-preserving). + let tag_lower = &lower[link_start..]; + let href_off = tag_lower.find("href=\"")?; + let value_start = link_start + href_off + 6; // past href=" + let value_end = html[value_start..].find('"')?; + let data_url = &html[value_start..value_start + value_end]; + + // Parse data:<mime>;base64,<payload> + let rest = data_url.strip_prefix("data:")?; + let comma = rest.find(',')?; + let meta = &rest[..comma]; + let b64 = &rest[comma + 1..]; + if !meta.to_ascii_lowercase().contains("base64") { + return None; + } + let mime = meta.split(';').next()?.trim().to_ascii_lowercase(); + let ext = mime_to_favicon_ext(&mime)?; + + let bytes = B64.decode(b64.trim()).ok()?; + let out = temp_dir.join(format!("{timestamp}.favicon{ext}")); + std::fs::write(&out, &bytes).ok()?; + hash_file(&out).ok().map(|h| (h, ext.to_string())) +} + +fn mime_to_favicon_ext(mime: &str) -> Option<&'static str> { + match mime { + "image/x-icon" | "image/vnd.microsoft.icon" => Some(".ico"), + "image/png" => Some(".png"), + "image/svg+xml" => Some(".svg"), + "image/jpeg" => Some(".jpg"), + "image/gif" => Some(".gif"), + "image/webp" => Some(".webp"), + _ => None, + } +} + #[cfg(test)] mod tests { use super::*;