1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

feat(core): auto-title for HTTP/S URL downloads

http::download now returns (hash, extension, Option<title_hint>).
Title is derived from Content-Disposition filename header (RFC 5987
filename*= preferred over plain filename=), falling back to the last
path segment of the final URL after redirects (percent-decoded).

capture.rs Source::Url arm passes the title through to record_media_entry
instead of None, so entries like 'Facharbeit.pdf' or 'facharbeit' appear
in the Title column instead of the raw entry_uid.

Adds percent_decode(), title_from_content_disposition(), title_from_url()
helpers with 10 new unit tests (122 total, all green).
This commit is contained in:
TheGeneralist 2026-06-24 14:56:48 +02:00
parent 03abfb4d18
commit 143f71bc17
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
2 changed files with 166 additions and 7 deletions

View file

@ -707,7 +707,7 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
// Source: generic HTTP/S file URL // Source: generic HTTP/S file URL
if source == Source::Url { if source == Source::Url {
match downloader::http::download(locator, store_path, &timestamp) { match downloader::http::download(locator, store_path, &timestamp) {
Ok((hash, file_extension)) => { Ok((hash, file_extension, title_hint)) => {
let temp_file = store_path let temp_file = store_path
.join("temp") .join("temp")
.join(&timestamp) .join(&timestamp)
@ -743,7 +743,7 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
&hash, &hash,
&file_extension, &file_extension,
byte_size, byte_size,
None, title_hint,
)?; )?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult { return Ok(CaptureResult {

View file

@ -3,15 +3,15 @@ use std::path::Path;
use crate::hash::hash_file; use crate::hash::hash_file;
/// Downloads a file from `url` into `store_path/temp/timestamp/`. /// Returns `(sha256_hex, extension_with_leading_dot, title_hint)` on success.
/// /// `title_hint` is derived from the `Content-Disposition` filename header, or the
/// Returns `(sha256_hex, extension_with_leading_dot)` on success. /// last path segment of the final URL after redirects.
/// ///
/// Errors if: /// Errors if:
/// - The request fails or returns a non-2xx status. /// - The request fails or returns a non-2xx status.
/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead). /// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
/// - The body cannot be written to disk. /// - The body cannot be written to disk.
pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String)> { pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String, Option<String>)> {
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10)) .redirect(reqwest::redirect::Policy::limited(10))
.user_agent("archivr/0.1") .user_agent("archivr/0.1")
@ -45,6 +45,16 @@ pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String
.or_else(|| extension_from_content_type(&content_type)) .or_else(|| extension_from_content_type(&content_type))
.unwrap_or_default(); .unwrap_or_default();
// Extract title before consuming the response body.
let title_hint = title_from_content_disposition(
response
.headers()
.get(reqwest::header::CONTENT_DISPOSITION)
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
)
.or_else(|| title_from_url(response.url().path()));
let temp_dir = store_path.join("temp").join(timestamp); let temp_dir = store_path.join("temp").join(timestamp);
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?; std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
@ -57,7 +67,7 @@ pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String
.with_context(|| format!("failed to write downloaded file to {}", out_file.display()))?; .with_context(|| format!("failed to write downloaded file to {}", out_file.display()))?;
let hash = hash_file(&out_file)?; let hash = hash_file(&out_file)?;
Ok((hash, extension)) Ok((hash, extension, title_hint))
} }
fn content_type_is_html(content_type: &str) -> bool { fn content_type_is_html(content_type: &str) -> bool {
@ -107,6 +117,87 @@ fn extension_from_content_type(content_type: &str) -> Option<String> {
} }
} }
/// Extracts a filename from a `Content-Disposition` header value.
///
/// Prefers `filename*=` (RFC 5987 percent-encoded, e.g. `filename*=UTF-8''Report%20Final.pdf`)
/// over plain `filename=`. Returns `None` if neither is present or the value is empty.
fn title_from_content_disposition(cd: &str) -> Option<String> {
// RFC 5987: filename*=charset'language'encoded
for part in cd.split(';') {
let part = part.trim();
if let Some(val) = part.strip_prefix("filename*=") {
let val = val.trim().trim_matches('"');
// encoded portion is after the second apostrophe
if let Some(encoded) = val.splitn(3, '\'').nth(2) {
let name = percent_decode(encoded);
if !name.is_empty() {
return Some(name);
}
}
}
}
// Plain filename=
for part in cd.split(';') {
let part = part.trim();
if let Some(val) = part.strip_prefix("filename=") {
let val = val.trim().trim_matches('"');
if !val.is_empty() {
return Some(val.to_string());
}
}
}
None
}
/// Derives a title from the last non-empty path segment of a URL path string.
///
/// Input is the raw percent-encoded path from the final URL after redirects
/// (e.g. `/papers/Facharbeit.pdf`). Returns `None` if the path has no meaningful segment.
fn title_from_url(path: &str) -> Option<String> {
let segment = path
.split('?')
.next()
.unwrap_or(path)
.rsplit('/')
.find(|s| !s.is_empty())?;
let decoded = percent_decode(segment);
if decoded.is_empty() { None } else { Some(decoded) }
}
/// Percent-decodes a string. Handles ASCII percent-encoded sequences; multi-byte
/// UTF-8 sequences are decoded correctly when each byte is encoded as `%XX`.
fn percent_decode(s: &str) -> String {
let mut bytes: Vec<u8> = Vec::with_capacity(s.len());
let src = s.as_bytes();
let mut i = 0;
while i < src.len() {
if src[i] == b'%' && i + 2 < src.len() {
let hi = src[i + 1];
let lo = src[i + 2];
let nibble = |b: u8| -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
};
if let (Some(h), Some(l)) = (nibble(hi), nibble(lo)) {
bytes.push((h << 4) | l);
i += 3;
continue;
}
}
if src[i] == b'+' {
bytes.push(b' ');
} else {
bytes.push(src[i]);
}
i += 1;
}
String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@ -160,4 +251,72 @@ mod tests {
fn content_type_is_html_pdf_is_not_html() { fn content_type_is_html_pdf_is_not_html() {
assert!(!content_type_is_html("application/pdf")); assert!(!content_type_is_html("application/pdf"));
} }
#[test]
fn title_from_url_plain_filename() {
assert_eq!(
title_from_url("/papers/Facharbeit.pdf"),
Some("Facharbeit.pdf".to_string())
);
}
#[test]
fn title_from_url_encoded() {
assert_eq!(
title_from_url("/files/My%20Report%202026.pdf"),
Some("My Report 2026.pdf".to_string())
);
}
#[test]
fn title_from_url_root_is_none() {
assert_eq!(title_from_url("/"), None);
}
#[test]
fn title_from_url_no_slash() {
assert_eq!(title_from_url("data.csv"), Some("data.csv".to_string()));
}
#[test]
fn title_from_content_disposition_plain() {
assert_eq!(
title_from_content_disposition("attachment; filename=\"Facharbeit.pdf\""),
Some("Facharbeit.pdf".to_string())
);
}
#[test]
fn title_from_content_disposition_rfc5987() {
assert_eq!(
title_from_content_disposition("attachment; filename*=UTF-8''Facharbeit%20Final.pdf"),
Some("Facharbeit Final.pdf".to_string())
);
}
#[test]
fn title_from_content_disposition_empty() {
assert_eq!(title_from_content_disposition(""), None);
}
#[test]
fn title_from_content_disposition_prefers_rfc5987() {
assert_eq!(
title_from_content_disposition(
"attachment; filename=\"old.pdf\"; filename*=UTF-8''new.pdf"
),
Some("new.pdf".to_string())
);
}
#[test]
fn percent_decode_utf8() {
// ä = 0xC3 0xA4
assert_eq!(percent_decode("%C3%A4"), "ä");
}
#[test]
fn percent_decode_plus_is_space() {
assert_eq!(percent_decode("hello+world"), "hello world");
}
} }