diff --git a/Cargo.lock b/Cargo.lock index d024ee2..f8f45f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,7 @@ dependencies = [ "serde", "serde_json", "sha3", + "tempfile", "uuid", ] diff --git a/crates/archivr-core/Cargo.toml b/crates/archivr-core/Cargo.toml index c446710..30ecc0f 100644 --- a/crates/archivr-core/Cargo.toml +++ b/crates/archivr-core/Cargo.toml @@ -14,3 +14,6 @@ serde_json.workspace = true sha3.workspace = true uuid.workspace = true reqwest = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/archivr-core/src/downloader/mod.rs b/crates/archivr-core/src/downloader/mod.rs index 21c6b34..d8c3911 100644 --- a/crates/archivr-core/src/downloader/mod.rs +++ b/crates/archivr-core/src/downloader/mod.rs @@ -4,3 +4,4 @@ pub mod tweets; pub mod ytdlp; pub mod metadata; pub mod http; +pub mod singlefile; diff --git a/crates/archivr-core/src/downloader/singlefile.rs b/crates/archivr-core/src/downloader/singlefile.rs new file mode 100644 index 0000000..06b409a --- /dev/null +++ b/crates/archivr-core/src/downloader/singlefile.rs @@ -0,0 +1,140 @@ +use anyhow::{Context, Result, bail}; +use std::{env, io::Read, path::Path, process::Command}; + +use crate::hash::hash_file; + +/// Archives `url` as a self-contained HTML snapshot. +/// +/// Returns `(sha256_hex, title_hint)` on success. +/// - `sha256_hex`: hash of the saved `.html` file, used as the blob key. +/// - `title_hint`: page title extracted from the `` tag, if present. +/// +/// Reads two env vars: +/// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`). +/// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`). +pub fn save(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, Option<String>)> { + let single_file = + env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string()); + let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string()); + save_with(url, store_path, timestamp, &single_file, &chrome) +} + +/// Inner implementation; takes binary paths explicitly so tests can inject them +/// without mutating process-global environment variables. +fn save_with( + url: &str, + store_path: &Path, + timestamp: &str, + single_file: &str, + chrome: &str, +) -> Result<(String, Option<String>)> { + let temp_dir = store_path.join("temp").join(timestamp); + std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?; + + let out_file = temp_dir.join(format!("{timestamp}.html")); + + let out = Command::new(single_file) + .arg(url) + .arg(&out_file) + .arg(format!("--browser-executable-path={chrome}")) + .arg("--browser-headless") + .output() + .with_context(|| format!("failed to spawn {single_file} process"))?; + + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + bail!("single-file failed: {stderr}"); + } + + if !out_file.exists() { + bail!( + "single-file exited successfully but produced no output file at {}", + out_file.display() + ); + } + + let title = extract_html_title(&out_file); + let hash = hash_file(&out_file)?; + Ok((hash, title)) +} + +/// Reads the first 8 KiB of `path` and extracts the content of the first +/// `<title>…` element. Returns `None` if absent or empty. +/// +/// Uses `to_ascii_lowercase` for case-insensitive tag matching. ASCII-only +/// lowercasing is byte-length-preserving, so byte offsets derived from the +/// lowercased buffer are valid indices into the original buffer. +fn extract_html_title(path: &Path) -> Option { + let mut buf = [0u8; 8192]; + let n = std::fs::File::open(path).ok()?.read(&mut buf).ok()?; + // Recover a valid UTF-8 prefix if the 8 KiB boundary falls mid-character. + let snippet = match std::str::from_utf8(&buf[..n]) { + Ok(s) => s, + Err(e) => std::str::from_utf8(&buf[..e.valid_up_to()]).ok()?, + }; + // ASCII-only lowercase: A-Z -> a-z, all other bytes unchanged. + // Byte lengths are identical to the original, so offsets are safe to reuse. + let lower = snippet.to_ascii_lowercase(); + let tag_start = lower.find("")?; + let content_start = tag_start + 7; // len("<title>") == 7 + let content_end = content_start + lower[content_start..].find("")?; + let title = snippet[content_start..content_end].trim(); + if title.is_empty() { None } else { Some(title.to_string()) } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn extract_html_title_finds_title() { + let mut f = NamedTempFile::new().unwrap(); + write!(f, "Paul Graham — Great Work").unwrap(); + assert_eq!( + extract_html_title(f.path()), + Some("Paul Graham — Great Work".to_string()) + ); + } + + #[test] + fn extract_html_title_case_insensitive() { + let mut f = NamedTempFile::new().unwrap(); + write!(f, "My Page").unwrap(); + assert_eq!(extract_html_title(f.path()), Some("My Page".to_string())); + } + + #[test] + fn extract_html_title_empty_tag_returns_none() { + let mut f = NamedTempFile::new().unwrap(); + write!(f, " ").unwrap(); + assert_eq!(extract_html_title(f.path()), None); + } + + #[test] + fn extract_html_title_no_title_tag_returns_none() { + let mut f = NamedTempFile::new().unwrap(); + write!(f, "no title here").unwrap(); + assert_eq!(extract_html_title(f.path()), None); + } + + #[test] + fn save_with_missing_binary_returns_clear_error() { + // Calls save_with directly — no env mutation, safe in parallel test runs. + let tmp = tempfile::tempdir().unwrap(); + let result = save_with( + "https://example.com", + tmp.path(), + "test-ts", + "/nonexistent/single-file", + "chromium", + ); + let err = result.unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("spawn") || msg.contains("nonexistent") || msg.contains("No such"), + "unexpected error: {msg}" + ); + } +}