1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

feat(core): generic HTTP/S file URL capture (Track 1)

- Add crates/archivr-core/src/downloader/http.rs
  - download(url, store_path, timestamp) -> Result<(hash, extension)>
  - Rejects text/html responses with a clear error
  - Derives extension from URL path or Content-Type header
  - Follows redirects (capped at 10), user-agent archivr/0.1
  - 10 unit tests for extension/content-type helpers
- Add Source::Url variant to capture::Source enum
- determine_source: unmatched http/https URLs route to Source::Url
- source_metadata: Source::Url => ("web", "file", "file")
- generate_entry_title: Source::Url arm -> "Downloaded File" fallback
- perform_capture: Source::Url arm calls http::download, uses existing
  temp -> hash_exists -> move_temp_to_raw -> record_media_entry pipeline
- Update test expectations: 3 plain https:// cases now expect Source::Url
- Add reqwest 0.12 (blocking) to workspace and archivr-core deps
- Mark URLs milestone done in docs/README.md
- Update NEXT.md Track 1 status
This commit is contained in:
TheGeneralist 2026-06-24 14:37:08 +02:00
parent a76dcf647f
commit 03abfb4d18
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
7 changed files with 1017 additions and 19 deletions

View file

@ -22,6 +22,7 @@ pub enum Source {
Reddit,
Snapchat,
Local,
Url,
Other,
}
@ -86,6 +87,7 @@ fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
),
Source::Snapchat => format!("Snap by {}", meta.author.as_deref().unwrap_or("unknown")),
Source::Local => meta.title.clone().unwrap_or_else(|| "Local File".to_string()),
Source::Url => meta.title.clone().unwrap_or_else(|| "Downloaded File".to_string()),
Source::Other => "Archived Content".to_string(),
}
}
@ -317,6 +319,8 @@ fn determine_source(path: &str) -> Source {
{
return Source::Snapchat;
}
// No platform matched — treat as a generic file URL
return Source::Url;
}
if Path::new(path).exists() {
return Source::Local;
@ -411,6 +415,7 @@ fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str)
Source::Reddit => ("reddit", "post", "video"),
Source::Snapchat => ("snapchat", "story", "video"),
Source::Local => ("local", "file", "file"),
Source::Url => ("web", "file", "file"),
Source::Other => ("other", "unknown", "unknown"),
}
}
@ -699,6 +704,64 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
));
}
// Source: generic HTTP/S file URL
if source == Source::Url {
match downloader::http::download(locator, store_path, &timestamp) {
Ok((hash, file_extension)) => {
let temp_file = store_path
.join("temp")
.join(&timestamp)
.join(format!("{timestamp}{file_extension}"));
let byte_size = fs::metadata(&temp_file)
.with_context(|| format!("failed to stat staged file {}", temp_file.display()))?
.len() as i64;
let hash_exists = hash_exists(&hash, &file_extension, store_path)?;
if hash_exists {
let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
} else {
move_temp_to_raw(
&store_path
.join("temp")
.join(&timestamp)
.join(format!("{timestamp}{file_extension}")),
&hash,
store_path,
)?;
let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
}
record_media_entry(
&conn,
store_path,
user_id,
&run,
&item,
locator,
locator,
source,
&hash,
&file_extension,
byte_size,
None,
)?;
database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult {
run_uid: run.run_uid.clone(),
status: "completed".to_string(),
});
}
Err(e) => {
return Err(fail_run(
&conn,
&run,
&item,
&format!("Failed to download URL: {e}"),
));
}
}
}
// Sources: Tweets or Twitter Threads
if matches!(source, Source::Tweet | Source::TweetThread) {
let tweet_id = match tweet_id_from_archive_path(locator) {
@ -1210,15 +1273,15 @@ mod tests {
},
TestCase {
url: "https://example.com/",
expected: Source::Other,
expected: Source::Url,
},
TestCase {
url: "https://example.com/?redirect=instagram.com/reel/ABC123",
expected: Source::Other,
expected: Source::Url,
},
TestCase {
url: "https://notfacebook.com/watch?v=123456",
expected: Source::Other,
expected: Source::Url,
},
];