feat(core): generic HTTP/S file URL capture (Track 1)

- Add crates/archivr-core/src/downloader/http.rs
  - download(url, store_path, timestamp) -> Result<(hash, extension)>
  - Rejects text/html responses with a clear error
  - Derives extension from URL path or Content-Type header
  - Follows redirects (capped at 10), user-agent archivr/0.1
  - 10 unit tests for extension/content-type helpers
- Add Source::Url variant to capture::Source enum
- determine_source: unmatched http/https URLs route to Source::Url
- source_metadata: Source::Url => ("web", "file", "file")
- generate_entry_title: Source::Url arm -> "Downloaded File" fallback
- perform_capture: Source::Url arm calls http::download, uses existing
  temp -> hash_exists -> move_temp_to_raw -> record_media_entry pipeline
- Update test expectations: 3 plain https:// cases now expect Source::Url
- Add reqwest 0.12 (blocking) to workspace and archivr-core deps
- Mark URLs milestone done in docs/README.md
- Update NEXT.md Track 1 status

Cargo.toml

@@ -27,3 +27,4 @@ toml = "0.8.19"
 tower = "0.5.1"
 tower-http = { version = "0.6.2", features = ["fs", "trace"] }
 uuid = { version = "1.18.1", features = ["v4"] }
+reqwest = { version = "0.12", features = ["blocking"] }

crates/archivr-core/Cargo.toml

@@ -13,3 +13,4 @@ serde.workspace = true
 serde_json.workspace = true
 sha3.workspace = true
 uuid.workspace = true
+reqwest = { workspace = true }

crates/archivr-core/src/capture.rs

@@ -22,6 +22,7 @@ pub enum Source {
     Reddit,
     Snapchat,
     Local,
+    Url,
     Other,
 }
 
@@ -86,6 +87,7 @@ fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
         ),
         Source::Snapchat => format!("Snap by {}", meta.author.as_deref().unwrap_or("unknown")),
         Source::Local => meta.title.clone().unwrap_or_else(|| "Local File".to_string()),
+        Source::Url => meta.title.clone().unwrap_or_else(|| "Downloaded File".to_string()),
         Source::Other => "Archived Content".to_string(),
     }
 }
@@ -317,6 +319,8 @@ fn determine_source(path: &str) -> Source {
         {
             return Source::Snapchat;
         }
+        // No platform matched — treat as a generic file URL
+        return Source::Url;
     }
     if Path::new(path).exists() {
         return Source::Local;
@@ -411,6 +415,7 @@ fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str)
         Source::Reddit => ("reddit", "post", "video"),
         Source::Snapchat => ("snapchat", "story", "video"),
         Source::Local => ("local", "file", "file"),
+        Source::Url => ("web", "file", "file"),
         Source::Other => ("other", "unknown", "unknown"),
     }
 }
@@ -699,6 +704,64 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result<Ca
         ));
     }
 
+    // Source: generic HTTP/S file URL
+    if source == Source::Url {
+        match downloader::http::download(locator, store_path, &timestamp) {
+            Ok((hash, file_extension)) => {
+                let temp_file = store_path
+                    .join("temp")
+                    .join(&timestamp)
+                    .join(format!("{timestamp}{file_extension}"));
+                let byte_size = fs::metadata(&temp_file)
+                    .with_context(|| format!("failed to stat staged file {}", temp_file.display()))?
+                    .len() as i64;
+
+                let hash_exists = hash_exists(&hash, &file_extension, store_path)?;
+                if hash_exists {
+                    let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
+                } else {
+                    move_temp_to_raw(
+                        &store_path
+                            .join("temp")
+                            .join(&timestamp)
+                            .join(format!("{timestamp}{file_extension}")),
+                        &hash,
+                        store_path,
+                    )?;
+                    let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
+                }
+
+                record_media_entry(
+                    &conn,
+                    store_path,
+                    user_id,
+                    &run,
+                    &item,
+                    locator,
+                    locator,
+                    source,
+                    &hash,
+                    &file_extension,
+                    byte_size,
+                    None,
+                )?;
+                database::finish_archive_run(&conn, run.id)?;
+                return Ok(CaptureResult {
+                    run_uid: run.run_uid.clone(),
+                    status: "completed".to_string(),
+                });
+            }
+            Err(e) => {
+                return Err(fail_run(
+                    &conn,
+                    &run,
+                    &item,
+                    &format!("Failed to download URL: {e}"),
+                ));
+            }
+        }
+    }
+
     // Sources: Tweets or Twitter Threads
     if matches!(source, Source::Tweet | Source::TweetThread) {
         let tweet_id = match tweet_id_from_archive_path(locator) {
@@ -1210,15 +1273,15 @@ mod tests {
             },
             TestCase {
                 url: "https://example.com/",
-                expected: Source::Other,
+                expected: Source::Url,
             },
             TestCase {
                 url: "https://example.com/?redirect=instagram.com/reel/ABC123",
-                expected: Source::Other,
+                expected: Source::Url,
             },
             TestCase {
                 url: "https://notfacebook.com/watch?v=123456",
-                expected: Source::Other,
+                expected: Source::Url,
             },
         ];
 

crates/archivr-core/src/downloader/http.rs

@@ -0,0 +1,163 @@
+use anyhow::{Context, Result, bail};
+use std::path::Path;
+
+use crate::hash::hash_file;
+
+/// Downloads a file from `url` into `store_path/temp/timestamp/`.
+///
+/// Returns `(sha256_hex, extension_with_leading_dot)` on success.
+///
+/// Errors if:
+/// - The request fails or returns a non-2xx status.
+/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
+/// - The body cannot be written to disk.
+pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String)> {
+    let client = reqwest::blocking::Client::builder()
+        .redirect(reqwest::redirect::Policy::limited(10))
+        .user_agent("archivr/0.1")
+        .build()
+        .context("failed to build HTTP client")?;
+
+    let response = client
+        .get(url)
+        .send()
+        .with_context(|| format!("failed to fetch {url}"))?;
+
+    if !response.status().is_success() {
+        bail!("HTTP {} fetching {url}", response.status());
+    }
+
+    let content_type = response
+        .headers()
+        .get(reqwest::header::CONTENT_TYPE)
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or("")
+        .to_string();
+
+    if content_type_is_html(&content_type) {
+        bail!(
+            "URL returned HTML (Content-Type: {content_type}); \
+             use a web-page archiver for this URL"
+        );
+    }
+
+    let extension = extension_from_url(url)
+        .or_else(|| extension_from_content_type(&content_type))
+        .unwrap_or_default();
+
+    let temp_dir = store_path.join("temp").join(timestamp);
+    std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
+
+    let out_file = temp_dir.join(format!("{timestamp}{extension}"));
+
+    let bytes = response
+        .bytes()
+        .with_context(|| format!("failed to read response body from {url}"))?;
+    std::fs::write(&out_file, &bytes)
+        .with_context(|| format!("failed to write downloaded file to {}", out_file.display()))?;
+
+    let hash = hash_file(&out_file)?;
+    Ok((hash, extension))
+}
+
+fn content_type_is_html(content_type: &str) -> bool {
+    let ct = content_type.split(';').next().unwrap_or("").trim();
+    ct == "text/html" || ct == "application/xhtml+xml"
+}
+
+/// Derives a file extension (e.g. `".pdf"`) from the URL path component.
+/// Strips query strings first. Returns `None` if no recognizable extension found.
+fn extension_from_url(url: &str) -> Option<String> {
+    let path = url.split('?').next().unwrap_or(url);
+    let last_segment = path.rsplit('/').next().unwrap_or("");
+    if let Some(dot_pos) = last_segment.rfind('.') {
+        let ext = &last_segment[dot_pos..];
+        // Accept only short, alphanumeric extensions (1–5 chars after the dot)
+        if ext.len() >= 2 && ext.len() <= 6 && ext[1..].chars().all(|c| c.is_ascii_alphanumeric()) {
+            return Some(ext.to_ascii_lowercase());
+        }
+    }
+    None
+}
+
+/// Maps a MIME type to a file extension. Returns `None` for unrecognized types.
+fn extension_from_content_type(content_type: &str) -> Option<String> {
+    let ct = content_type.split(';').next().unwrap_or("").trim();
+    match ct {
+        "application/pdf" => Some(".pdf".to_string()),
+        "image/jpeg" | "image/jpg" => Some(".jpg".to_string()),
+        "image/png" => Some(".png".to_string()),
+        "image/gif" => Some(".gif".to_string()),
+        "image/webp" => Some(".webp".to_string()),
+        "image/svg+xml" => Some(".svg".to_string()),
+        "video/mp4" => Some(".mp4".to_string()),
+        "video/webm" => Some(".webm".to_string()),
+        "video/ogg" => Some(".ogv".to_string()),
+        "audio/mpeg" | "audio/mp3" => Some(".mp3".to_string()),
+        "audio/ogg" => Some(".ogg".to_string()),
+        "audio/wav" => Some(".wav".to_string()),
+        "application/zip" => Some(".zip".to_string()),
+        "application/gzip" => Some(".gz".to_string()),
+        "application/json" => Some(".json".to_string()),
+        "text/plain" => Some(".txt".to_string()),
+        "text/csv" => Some(".csv".to_string()),
+        "text/xml" | "application/xml" => Some(".xml".to_string()),
+        "application/epub+zip" => Some(".epub".to_string()),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn extension_from_url_pdf() {
+        assert_eq!(extension_from_url("https://example.com/paper.pdf"), Some(".pdf".to_string()));
+    }
+
+    #[test]
+    fn extension_from_url_strips_query() {
+        assert_eq!(extension_from_url("https://example.com/file.zip?token=abc"), Some(".zip".to_string()));
+    }
+
+    #[test]
+    fn extension_from_url_no_extension() {
+        assert_eq!(extension_from_url("https://example.com/page"), None);
+    }
+
+    #[test]
+    fn extension_from_url_rejects_long_ext() {
+        assert_eq!(extension_from_url("https://example.com/file.toolongext"), None);
+    }
+
+    #[test]
+    fn extension_from_content_type_pdf() {
+        assert_eq!(extension_from_content_type("application/pdf"), Some(".pdf".to_string()));
+    }
+
+    #[test]
+    fn extension_from_content_type_with_params() {
+        assert_eq!(extension_from_content_type("application/pdf; charset=utf-8"), Some(".pdf".to_string()));
+    }
+
+    #[test]
+    fn content_type_is_html_plain() {
+        assert!(content_type_is_html("text/html"));
+    }
+
+    #[test]
+    fn content_type_is_html_with_charset() {
+        assert!(content_type_is_html("text/html; charset=utf-8"));
+    }
+
+    #[test]
+    fn content_type_is_html_xhtml() {
+        assert!(content_type_is_html("application/xhtml+xml"));
+    }
+
+    #[test]
+    fn content_type_is_html_pdf_is_not_html() {
+        assert!(!content_type_is_html("application/pdf"));
+    }
+}

crates/archivr-core/src/downloader/mod.rs

@@ -3,3 +3,4 @@ pub mod store;
 pub mod tweets;
 pub mod ytdlp;
 pub mod metadata;
+pub mod http;

docs/README.md

@@ -17,7 +17,7 @@ An open-source self-hosted archiving tool. Work in progress.
   - [x] Archiving local files
   - [x] Archiving Twitter Tweets, Threads, and Articles
   - [ ] Archiving files from cloud storage services (Google Drive, Dropbox, OneDrive) and from URLs
-    - [ ] URLs
+    - [x] URLs
     - [ ] Google Drive
     - [ ] Dropbox
     - [ ] OneDrive