mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat(core): auto-title for HTTP/S URL downloads
http::download now returns (hash, extension, Option<title_hint>). Title is derived from Content-Disposition filename header (RFC 5987 filename*= preferred over plain filename=), falling back to the last path segment of the final URL after redirects (percent-decoded). capture.rs Source::Url arm passes the title through to record_media_entry instead of None, so entries like 'Facharbeit.pdf' or 'facharbeit' appear in the Title column instead of the raw entry_uid. Adds percent_decode(), title_from_content_disposition(), title_from_url() helpers with 10 new unit tests (122 total, all green).
This commit is contained in:
parent
03abfb4d18
commit
143f71bc17
2 changed files with 166 additions and 7 deletions
|
|
@ -707,7 +707,7 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
|
||||||
// Source: generic HTTP/S file URL
|
// Source: generic HTTP/S file URL
|
||||||
if source == Source::Url {
|
if source == Source::Url {
|
||||||
match downloader::http::download(locator, store_path, ×tamp) {
|
match downloader::http::download(locator, store_path, ×tamp) {
|
||||||
Ok((hash, file_extension)) => {
|
Ok((hash, file_extension, title_hint)) => {
|
||||||
let temp_file = store_path
|
let temp_file = store_path
|
||||||
.join("temp")
|
.join("temp")
|
||||||
.join(×tamp)
|
.join(×tamp)
|
||||||
|
|
@ -743,7 +743,7 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
|
||||||
&hash,
|
&hash,
|
||||||
&file_extension,
|
&file_extension,
|
||||||
byte_size,
|
byte_size,
|
||||||
None,
|
title_hint,
|
||||||
)?;
|
)?;
|
||||||
database::finish_archive_run(&conn, run.id)?;
|
database::finish_archive_run(&conn, run.id)?;
|
||||||
return Ok(CaptureResult {
|
return Ok(CaptureResult {
|
||||||
|
|
|
||||||
|
|
@ -3,15 +3,15 @@ use std::path::Path;
|
||||||
|
|
||||||
use crate::hash::hash_file;
|
use crate::hash::hash_file;
|
||||||
|
|
||||||
/// Downloads a file from `url` into `store_path/temp/timestamp/`.
|
/// Returns `(sha256_hex, extension_with_leading_dot, title_hint)` on success.
|
||||||
///
|
/// `title_hint` is derived from the `Content-Disposition` filename header, or the
|
||||||
/// Returns `(sha256_hex, extension_with_leading_dot)` on success.
|
/// last path segment of the final URL after redirects.
|
||||||
///
|
///
|
||||||
/// Errors if:
|
/// Errors if:
|
||||||
/// - The request fails or returns a non-2xx status.
|
/// - The request fails or returns a non-2xx status.
|
||||||
/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
|
/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
|
||||||
/// - The body cannot be written to disk.
|
/// - The body cannot be written to disk.
|
||||||
pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String)> {
|
pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String, Option<String>)> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder()
|
||||||
.redirect(reqwest::redirect::Policy::limited(10))
|
.redirect(reqwest::redirect::Policy::limited(10))
|
||||||
.user_agent("archivr/0.1")
|
.user_agent("archivr/0.1")
|
||||||
|
|
@ -45,6 +45,16 @@ pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String
|
||||||
.or_else(|| extension_from_content_type(&content_type))
|
.or_else(|| extension_from_content_type(&content_type))
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Extract title before consuming the response body.
|
||||||
|
let title_hint = title_from_content_disposition(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get(reqwest::header::CONTENT_DISPOSITION)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or(""),
|
||||||
|
)
|
||||||
|
.or_else(|| title_from_url(response.url().path()));
|
||||||
|
|
||||||
let temp_dir = store_path.join("temp").join(timestamp);
|
let temp_dir = store_path.join("temp").join(timestamp);
|
||||||
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
|
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
|
||||||
|
|
||||||
|
|
@ -57,7 +67,7 @@ pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String
|
||||||
.with_context(|| format!("failed to write downloaded file to {}", out_file.display()))?;
|
.with_context(|| format!("failed to write downloaded file to {}", out_file.display()))?;
|
||||||
|
|
||||||
let hash = hash_file(&out_file)?;
|
let hash = hash_file(&out_file)?;
|
||||||
Ok((hash, extension))
|
Ok((hash, extension, title_hint))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn content_type_is_html(content_type: &str) -> bool {
|
fn content_type_is_html(content_type: &str) -> bool {
|
||||||
|
|
@ -107,6 +117,87 @@ fn extension_from_content_type(content_type: &str) -> Option<String> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Extracts a filename from a `Content-Disposition` header value.
|
||||||
|
///
|
||||||
|
/// Prefers `filename*=` (RFC 5987 percent-encoded, e.g. `filename*=UTF-8''Report%20Final.pdf`)
|
||||||
|
/// over plain `filename=`. Returns `None` if neither is present or the value is empty.
|
||||||
|
fn title_from_content_disposition(cd: &str) -> Option<String> {
|
||||||
|
// RFC 5987: filename*=charset'language'encoded
|
||||||
|
for part in cd.split(';') {
|
||||||
|
let part = part.trim();
|
||||||
|
if let Some(val) = part.strip_prefix("filename*=") {
|
||||||
|
let val = val.trim().trim_matches('"');
|
||||||
|
// encoded portion is after the second apostrophe
|
||||||
|
if let Some(encoded) = val.splitn(3, '\'').nth(2) {
|
||||||
|
let name = percent_decode(encoded);
|
||||||
|
if !name.is_empty() {
|
||||||
|
return Some(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Plain filename=
|
||||||
|
for part in cd.split(';') {
|
||||||
|
let part = part.trim();
|
||||||
|
if let Some(val) = part.strip_prefix("filename=") {
|
||||||
|
let val = val.trim().trim_matches('"');
|
||||||
|
if !val.is_empty() {
|
||||||
|
return Some(val.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Derives a title from the last non-empty path segment of a URL path string.
|
||||||
|
///
|
||||||
|
/// Input is the raw percent-encoded path from the final URL after redirects
|
||||||
|
/// (e.g. `/papers/Facharbeit.pdf`). Returns `None` if the path has no meaningful segment.
|
||||||
|
fn title_from_url(path: &str) -> Option<String> {
|
||||||
|
let segment = path
|
||||||
|
.split('?')
|
||||||
|
.next()
|
||||||
|
.unwrap_or(path)
|
||||||
|
.rsplit('/')
|
||||||
|
.find(|s| !s.is_empty())?;
|
||||||
|
let decoded = percent_decode(segment);
|
||||||
|
if decoded.is_empty() { None } else { Some(decoded) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Percent-decodes a string. Handles ASCII percent-encoded sequences; multi-byte
|
||||||
|
/// UTF-8 sequences are decoded correctly when each byte is encoded as `%XX`.
|
||||||
|
fn percent_decode(s: &str) -> String {
|
||||||
|
let mut bytes: Vec<u8> = Vec::with_capacity(s.len());
|
||||||
|
let src = s.as_bytes();
|
||||||
|
let mut i = 0;
|
||||||
|
while i < src.len() {
|
||||||
|
if src[i] == b'%' && i + 2 < src.len() {
|
||||||
|
let hi = src[i + 1];
|
||||||
|
let lo = src[i + 2];
|
||||||
|
let nibble = |b: u8| -> Option<u8> {
|
||||||
|
match b {
|
||||||
|
b'0'..=b'9' => Some(b - b'0'),
|
||||||
|
b'a'..=b'f' => Some(b - b'a' + 10),
|
||||||
|
b'A'..=b'F' => Some(b - b'A' + 10),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let (Some(h), Some(l)) = (nibble(hi), nibble(lo)) {
|
||||||
|
bytes.push((h << 4) | l);
|
||||||
|
i += 3;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if src[i] == b'+' {
|
||||||
|
bytes.push(b' ');
|
||||||
|
} else {
|
||||||
|
bytes.push(src[i]);
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -160,4 +251,72 @@ mod tests {
|
||||||
fn content_type_is_html_pdf_is_not_html() {
|
fn content_type_is_html_pdf_is_not_html() {
|
||||||
assert!(!content_type_is_html("application/pdf"));
|
assert!(!content_type_is_html("application/pdf"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_url_plain_filename() {
|
||||||
|
assert_eq!(
|
||||||
|
title_from_url("/papers/Facharbeit.pdf"),
|
||||||
|
Some("Facharbeit.pdf".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_url_encoded() {
|
||||||
|
assert_eq!(
|
||||||
|
title_from_url("/files/My%20Report%202026.pdf"),
|
||||||
|
Some("My Report 2026.pdf".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_url_root_is_none() {
|
||||||
|
assert_eq!(title_from_url("/"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_url_no_slash() {
|
||||||
|
assert_eq!(title_from_url("data.csv"), Some("data.csv".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_content_disposition_plain() {
|
||||||
|
assert_eq!(
|
||||||
|
title_from_content_disposition("attachment; filename=\"Facharbeit.pdf\""),
|
||||||
|
Some("Facharbeit.pdf".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_content_disposition_rfc5987() {
|
||||||
|
assert_eq!(
|
||||||
|
title_from_content_disposition("attachment; filename*=UTF-8''Facharbeit%20Final.pdf"),
|
||||||
|
Some("Facharbeit Final.pdf".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_content_disposition_empty() {
|
||||||
|
assert_eq!(title_from_content_disposition(""), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn title_from_content_disposition_prefers_rfc5987() {
|
||||||
|
assert_eq!(
|
||||||
|
title_from_content_disposition(
|
||||||
|
"attachment; filename=\"old.pdf\"; filename*=UTF-8''new.pdf"
|
||||||
|
),
|
||||||
|
Some("new.pdf".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn percent_decode_utf8() {
|
||||||
|
// ä = 0xC3 0xA4
|
||||||
|
assert_eq!(percent_decode("%C3%A4"), "ä");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn percent_decode_plus_is_space() {
|
||||||
|
assert_eq!(percent_decode("hello+world"), "hello world");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue