mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat(core): generic HTTP/S file URL capture (Track 1)
- Add crates/archivr-core/src/downloader/http.rs
- download(url, store_path, timestamp) -> Result<(hash, extension)>
- Rejects text/html responses with a clear error
- Derives extension from URL path or Content-Type header
- Follows redirects (capped at 10), user-agent archivr/0.1
- 10 unit tests for extension/content-type helpers
- Add Source::Url variant to capture::Source enum
- determine_source: unmatched http/https URLs route to Source::Url
- source_metadata: Source::Url => ("web", "file", "file")
- generate_entry_title: Source::Url arm -> "Downloaded File" fallback
- perform_capture: Source::Url arm calls http::download, uses existing
temp -> hash_exists -> move_temp_to_raw -> record_media_entry pipeline
- Update test expectations: 3 plain https:// cases now expect Source::Url
- Add reqwest 0.12 (blocking) to workspace and archivr-core deps
- Mark URLs milestone done in docs/README.md
- Update NEXT.md Track 1 status
This commit is contained in:
parent
a76dcf647f
commit
03abfb4d18
7 changed files with 1017 additions and 19 deletions
|
|
@ -22,6 +22,7 @@ pub enum Source {
|
|||
Reddit,
|
||||
Snapchat,
|
||||
Local,
|
||||
Url,
|
||||
Other,
|
||||
}
|
||||
|
||||
|
|
@ -86,6 +87,7 @@ fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
|
|||
),
|
||||
Source::Snapchat => format!("Snap by {}", meta.author.as_deref().unwrap_or("unknown")),
|
||||
Source::Local => meta.title.clone().unwrap_or_else(|| "Local File".to_string()),
|
||||
Source::Url => meta.title.clone().unwrap_or_else(|| "Downloaded File".to_string()),
|
||||
Source::Other => "Archived Content".to_string(),
|
||||
}
|
||||
}
|
||||
|
|
@ -317,6 +319,8 @@ fn determine_source(path: &str) -> Source {
|
|||
{
|
||||
return Source::Snapchat;
|
||||
}
|
||||
// No platform matched — treat as a generic file URL
|
||||
return Source::Url;
|
||||
}
|
||||
if Path::new(path).exists() {
|
||||
return Source::Local;
|
||||
|
|
@ -411,6 +415,7 @@ fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str)
|
|||
Source::Reddit => ("reddit", "post", "video"),
|
||||
Source::Snapchat => ("snapchat", "story", "video"),
|
||||
Source::Local => ("local", "file", "file"),
|
||||
Source::Url => ("web", "file", "file"),
|
||||
Source::Other => ("other", "unknown", "unknown"),
|
||||
}
|
||||
}
|
||||
|
|
@ -699,6 +704,64 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
|
|||
));
|
||||
}
|
||||
|
||||
// Source: generic HTTP/S file URL
|
||||
if source == Source::Url {
|
||||
match downloader::http::download(locator, store_path, ×tamp) {
|
||||
Ok((hash, file_extension)) => {
|
||||
let temp_file = store_path
|
||||
.join("temp")
|
||||
.join(×tamp)
|
||||
.join(format!("{timestamp}{file_extension}"));
|
||||
let byte_size = fs::metadata(&temp_file)
|
||||
.with_context(|| format!("failed to stat staged file {}", temp_file.display()))?
|
||||
.len() as i64;
|
||||
|
||||
let hash_exists = hash_exists(&hash, &file_extension, store_path)?;
|
||||
if hash_exists {
|
||||
let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp));
|
||||
} else {
|
||||
move_temp_to_raw(
|
||||
&store_path
|
||||
.join("temp")
|
||||
.join(×tamp)
|
||||
.join(format!("{timestamp}{file_extension}")),
|
||||
&hash,
|
||||
store_path,
|
||||
)?;
|
||||
let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp));
|
||||
}
|
||||
|
||||
record_media_entry(
|
||||
&conn,
|
||||
store_path,
|
||||
user_id,
|
||||
&run,
|
||||
&item,
|
||||
locator,
|
||||
locator,
|
||||
source,
|
||||
&hash,
|
||||
&file_extension,
|
||||
byte_size,
|
||||
None,
|
||||
)?;
|
||||
database::finish_archive_run(&conn, run.id)?;
|
||||
return Ok(CaptureResult {
|
||||
run_uid: run.run_uid.clone(),
|
||||
status: "completed".to_string(),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(fail_run(
|
||||
&conn,
|
||||
&run,
|
||||
&item,
|
||||
&format!("Failed to download URL: {e}"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sources: Tweets or Twitter Threads
|
||||
if matches!(source, Source::Tweet | Source::TweetThread) {
|
||||
let tweet_id = match tweet_id_from_archive_path(locator) {
|
||||
|
|
@ -1210,15 +1273,15 @@ mod tests {
|
|||
},
|
||||
TestCase {
|
||||
url: "https://example.com/",
|
||||
expected: Source::Other,
|
||||
expected: Source::Url,
|
||||
},
|
||||
TestCase {
|
||||
url: "https://example.com/?redirect=instagram.com/reel/ABC123",
|
||||
expected: Source::Other,
|
||||
expected: Source::Url,
|
||||
},
|
||||
TestCase {
|
||||
url: "https://notfacebook.com/watch?v=123456",
|
||||
expected: Source::Other,
|
||||
expected: Source::Url,
|
||||
},
|
||||
];
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue