diff --git a/crates/archivr-core/src/database.rs b/crates/archivr-core/src/database.rs index d207c35..69521dc 100644 --- a/crates/archivr-core/src/database.rs +++ b/crates/archivr-core/src/database.rs @@ -431,6 +431,26 @@ pub fn upsert_blob(conn: &Connection, blob: &BlobRecord) -> Result { Ok(id) } +/// Returns the `BlobRecord` for the given SHA-256 hex digest, or `None` if not found. +pub fn get_blob_by_sha256(conn: &Connection, sha256: &str) -> Result> { + conn.query_row( + "SELECT sha256, byte_size, mime_type, extension, raw_relpath + FROM blobs WHERE sha256 = ?1", + [sha256], + |row| { + Ok(BlobRecord { + sha256: row.get(0)?, + byte_size: row.get(1)?, + mime_type: row.get(2)?, + extension: row.get(3)?, + raw_relpath: row.get(4)?, + }) + }, + ) + .optional() + .map_err(anyhow::Error::from) +} + pub fn create_archived_entry(conn: &Connection, entry: &NewEntry) -> Result { validate_visibility(&entry.visibility)?; let entry_uid = public_id("entry"); @@ -1101,4 +1121,32 @@ mod tests { ); assert_eq!(entry_count_for_tag_path(&conn, "/sciences").unwrap(), 1); } + + #[test] + fn get_blob_by_sha256_round_trips() { + let conn = conn(); + let blob = BlobRecord { + sha256: "deadbeef01234567".repeat(4), // 64-char hex string + byte_size: 1234, + mime_type: Some("font/woff2".to_string()), + extension: Some("woff2".to_string()), + raw_relpath: "raw/d/e/deadbeef.woff2".to_string(), + }; + upsert_blob(&conn, &blob).unwrap(); + + let found = get_blob_by_sha256(&conn, &blob.sha256).unwrap(); + assert!(found.is_some(), "should find the blob we just upserted"); + let found = found.unwrap(); + assert_eq!(found.sha256, blob.sha256); + assert_eq!(found.byte_size, 1234); + assert_eq!(found.mime_type, Some("font/woff2".to_string())); + assert_eq!(found.raw_relpath, blob.raw_relpath); + } + + #[test] + fn get_blob_by_sha256_returns_none_for_unknown() { + let conn = conn(); + let result = get_blob_by_sha256(&conn, "0000000000000000000000000000000000000000000000000000000000000000").unwrap(); + assert!(result.is_none()); + } } diff --git a/crates/archivr-core/src/downloader/font_extractor.rs b/crates/archivr-core/src/downloader/font_extractor.rs new file mode 100644 index 0000000..3ae8ecc --- /dev/null +++ b/crates/archivr-core/src/downloader/font_extractor.rs @@ -0,0 +1,187 @@ +use anyhow::Result; +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine as _; +use regex::Regex; +use std::{fs, path::Path}; + +use crate::hash::hash_bytes; + +#[derive(Debug, Clone)] +pub struct ExtractedFont { + /// SHA3-256 hex of the raw font bytes. + pub sha256: String, + /// File extension including leading dot, e.g. `".woff2"`. + pub ext: String, + /// Size in bytes of the decoded font. + pub byte_size: i64, + /// Store-relative path, e.g. `"raw/a/b/abc...woff2"`. + pub raw_relpath: String, +} + +/// Scans `html` for `@font-face` `src` attributes containing `data:font/...;base64,...` +/// URLs, decodes each font, stores it in `{store_path}/raw/`, and rewrites the src +/// to `/api/archives/{archive_id}/blobs/{sha256}`. +/// +/// Returns `(rewritten_html, Vec)`. Each occurrence is reported +/// individually; the caller deduplicates via `upsert_blob`. +pub fn extract_and_rewrite( + html: &str, + store_path: &Path, + archive_id: &str, +) -> Result<(String, Vec)> { + // Matches: url(data:font/MIME;base64,DATA) or url("data:font/MIME;base64,DATA") + // Also matches data:application/font-... for older MIME types. + // Group 2 captures the full MIME type (e.g. "font/woff2"), group 3 the base64 payload. + let re = Regex::new( + r#"url\("?(data:((?:font|application)/[^;]+);base64,([A-Za-z0-9+/=]+))"?\)"#, + )?; + + let mut fonts = Vec::new(); + let rewritten = re.replace_all(html, |caps: ®ex::Captures| { + let mime = caps[2].to_ascii_lowercase(); + let b64_data = &caps[3]; + + let ext = match mime_to_ext(&mime) { + Some(e) => e, + None => return caps[0].to_string(), // unknown MIME — leave as-is + }; + + let bytes = match B64.decode(b64_data) { + Ok(b) => b, + Err(_) => return caps[0].to_string(), // corrupt base64 — leave as-is + }; + + let sha256 = hash_bytes(&bytes); + let raw_relpath = font_raw_relpath(&sha256, ext); + let abs_path = store_path.join(&raw_relpath); + + if !abs_path.exists() { + if let Some(parent) = abs_path.parent() { + let _ = fs::create_dir_all(parent); + } + if fs::write(&abs_path, &bytes).is_err() { + return caps[0].to_string(); // write failed — leave as-is + } + } + + fonts.push(ExtractedFont { + sha256: sha256.clone(), + ext: ext.to_string(), + byte_size: bytes.len() as i64, + raw_relpath, + }); + + format!("url(/api/archives/{archive_id}/blobs/{sha256})") + }); + + Ok((rewritten.into_owned(), fonts)) +} + +fn font_raw_relpath(sha256: &str, ext: &str) -> String { + let mut chars = sha256.chars(); + let a = chars.next().unwrap_or('0'); + let b = chars.next().unwrap_or('0'); + format!("raw/{a}/{b}/{sha256}{ext}") +} + +fn mime_to_ext(mime: &str) -> Option<&'static str> { + match mime { + "font/woff2" | "application/font-woff2" => Some(".woff2"), + "font/woff" | "application/font-woff" => Some(".woff"), + "font/ttf" | "font/truetype" | "application/x-font-truetype" => Some(".ttf"), + "font/otf" | "font/opentype" | "application/x-font-opentype" => Some(".otf"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn html_with_woff2_font() -> String { + let font_b64 = B64.encode(b"WOFF2FAKEDATA"); + format!( + "\ +

Hello

" + ) + } + + #[test] + fn replaces_data_url_with_api_url() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("raw")).unwrap(); + + let html = html_with_woff2_font(); + let (rewritten, fonts) = extract_and_rewrite(&html, tmp.path(), "myarchive").unwrap(); + + assert_eq!(fonts.len(), 1, "should extract one font"); + let font = &fonts[0]; + assert_eq!(font.ext, ".woff2"); + assert_eq!(font.byte_size, b"WOFF2FAKEDATA".len() as i64); + assert!(!font.sha256.is_empty()); + assert!(!rewritten.contains("data:font"), "data URL must be gone from HTML"); + assert!( + rewritten.contains(&format!("/api/archives/myarchive/blobs/{}", font.sha256)), + "rewritten HTML must contain the local API URL" + ); + } + + #[test] + fn font_file_is_written_to_raw_store() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("raw")).unwrap(); + + let html = html_with_woff2_font(); + let (_, fonts) = extract_and_rewrite(&html, tmp.path(), "myarchive").unwrap(); + + let font = &fonts[0]; + let raw_path = tmp.path().join(&font.raw_relpath); + assert!(raw_path.exists(), "font file must exist at raw_relpath"); + let written = std::fs::read(&raw_path).unwrap(); + assert_eq!(written, b"WOFF2FAKEDATA"); + } + + #[test] + fn deduplicates_identical_fonts() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("raw")).unwrap(); + + let font_b64 = B64.encode(b"SAMEDATA"); + let html = format!( + "" + ); + let (_, fonts) = extract_and_rewrite(&html, tmp.path(), "x").unwrap(); + assert_eq!(fonts.len(), 2); + assert_eq!(fonts[0].sha256, fonts[1].sha256); + let raw_path = tmp.path().join(&fonts[0].raw_relpath); + assert!(raw_path.exists()); + } + + #[test] + fn html_without_font_data_urls_is_unchanged() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("raw")).unwrap(); + + let html = "

no fonts

"; + let (rewritten, fonts) = extract_and_rewrite(html, tmp.path(), "x").unwrap(); + assert_eq!(fonts.len(), 0); + assert_eq!(rewritten, html); + } + + #[test] + fn ttf_font_gets_correct_extension() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("raw")).unwrap(); + + let font_b64 = B64.encode(b"TTFDATA"); + let html = format!( + "" + ); + let (_, fonts) = extract_and_rewrite(&html, tmp.path(), "x").unwrap(); + assert_eq!(fonts[0].ext, ".ttf"); + } +} diff --git a/crates/archivr-core/src/downloader/mod.rs b/crates/archivr-core/src/downloader/mod.rs index d8c3911..6c8d62a 100644 --- a/crates/archivr-core/src/downloader/mod.rs +++ b/crates/archivr-core/src/downloader/mod.rs @@ -5,3 +5,4 @@ pub mod ytdlp; pub mod metadata; pub mod http; pub mod singlefile; +pub mod font_extractor; diff --git a/crates/archivr-core/src/hash.rs b/crates/archivr-core/src/hash.rs index cbf1194..949a86b 100644 --- a/crates/archivr-core/src/hash.rs +++ b/crates/archivr-core/src/hash.rs @@ -1,17 +1,56 @@ use anyhow::Result; use sha3::{Digest, Sha3_256}; -use std::{fs::File, io::Read, path::Path}; +use std::path::Path; -pub fn hash_file(path: &Path) -> Result { - let mut file = File::open(path)?; - let mut buf = Vec::new(); - file.read_to_end(&mut buf)?; - Ok(hash(String::from_utf8_lossy(&buf).to_string())) -} - -pub fn hash(path: String) -> String { +/// Computes the SHA3-256 hex digest of `data` directly on raw bytes. +pub fn hash_bytes(data: &[u8]) -> String { let mut hasher = Sha3_256::new(); - hasher.update(path.as_bytes()); - let result = hasher.finalize(); - hex::encode(result) + hasher.update(data); + hex::encode(hasher.finalize()) +} + +/// Reads `path` fully and returns its SHA3-256 hex digest. +/// Uses raw bytes — correct for both text and binary files. +pub fn hash_file(path: &Path) -> Result { + let buf = std::fs::read(path)?; + Ok(hash_bytes(&buf)) +} + +/// Hashes an arbitrary string (used for legacy callers; prefer `hash_bytes`). +pub fn hash(s: String) -> String { + hash_bytes(s.as_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hash_bytes_and_hash_file_agree_on_same_data() { + let data = b"hello archivr fonts"; + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), data).unwrap(); + assert_eq!(hash_bytes(data), hash_file(tmp.path()).unwrap()); + } + + #[test] + fn hash_bytes_is_stable() { + let h = hash_bytes(b"archivr"); + assert_eq!(h.len(), 64); + assert!(h.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn hash_bytes_differs_from_hash_of_lossy_utf8_for_binary_data() { + let binary = b"\xff\xfe\xfd"; + let direct = hash_bytes(binary); + let lossy = String::from_utf8_lossy(binary).to_string(); + let via_old_path = { + use sha3::{Digest, Sha3_256}; + let mut h = Sha3_256::new(); + h.update(lossy.as_bytes()); + hex::encode(h.finalize()) + }; + assert_ne!(direct, via_old_path); + } }