diff --git a/.gitignore b/.gitignore index 8f160bd..817505d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ !.gitignore -!docs -!docs/** +!docs/ +!docs/LICENSE +!docs/README* !crates !crates/** diff --git a/ARCHIVR-MENTAL-MODEL.md b/ARCHIVR-MENTAL-MODEL.md index 01f4387..f383ff1 100644 --- a/ARCHIVR-MENTAL-MODEL.md +++ b/ARCHIVR-MENTAL-MODEL.md @@ -2,6 +2,13 @@ This document explains the current project shape after the workspace refactor. +## Key Documents + +| Document | Role | +|---|---| +| `ARCHIVR-MENTAL-MODEL.md` | **This file.** Current architecture, data flows, and where to edit. | +| `docs/README.md` | User-facing docs: how to run the tool, supported inputs, environment variables. | + ## The Big Model Archivr is now a Rust workspace with three crates: @@ -57,6 +64,40 @@ label = "Personal" archive_path = "/path/to/archive/.archivr" ``` +## How To Run It + +There are two user-facing binaries: + +| Binary | Purpose | +|---|---| +| `archivr` | CLI for initializing archives and capturing material into one archive | +| `archivr-server` | Web server for browsing one or more existing archives | + +The CLI writes archive data: + +```sh +nix run .#archivr -- init ./my-archive --name "My Archive" +nix run .#archivr -- archive file:///absolute/path/to/file.pdf +``` + +The server reads archive data: + +```sh +nix run .#archivr-server -- ./archivr-server.toml +``` + +If no config path is passed, the server reads `./archivr-server.toml`. +The config is a server registry, not archive data: + +```toml +[[archives]] +id = "personal" +label = "Personal" +archive_path = "/absolute/path/to/my-archive/.archivr" +``` + +The packaged Nix server wrapper sets `ARCHIVR_STATIC_DIR` so the server can find the installed web UI assets. Source-tree runs do not need that variable because they fall back to `crates/archivr-server/static`. + ## Write Data Flow When archiving something through the CLI: @@ -130,6 +171,6 @@ The web server reads archive data and serves the UI. It does not yet implement c Search is currently simple client-side filtering. -Auth is not a production model yet. +**Auth and session model:** The server binds to `127.0.0.1` by default and has no authentication middleware. This is intentional — Archivr is a local tool. The bind address is configurable via the TOML `bind` field or `ARCHIVR_BIND` env var; a non-loopback address triggers a startup warning. Route families are classified (READ / ADMIN / WRITE / STATIC) in `crates/archivr-server/src/routes.rs` as the decision record for when middleware is eventually added. See the "Security and Deployment" section in `docs/README.md`. Admin is a mounted-archives view, not a management system. diff --git a/Cargo.lock b/Cargo.lock index 5a6b629..e220753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,6 +124,7 @@ dependencies = [ "archivr-core", "axum", "serde", + "serde_json", "tempfile", "tokio", "toml", diff --git a/crates/archivr-cli/src/main.rs b/crates/archivr-cli/src/main.rs index d47c9c9..4cf56a4 100644 --- a/crates/archivr-cli/src/main.rs +++ b/crates/archivr-cli/src/main.rs @@ -1,12 +1,9 @@ use anyhow::{Context, Result}; -use archivr_core::{archive, database, downloader, twitter::parse_tweet_id}; -use chrono::Local; +use archivr_core::archive; use clap::{Parser, Subcommand}; -use serde_json::json; use std::{ - collections::HashSet, - env, fs, - path::{Path, PathBuf}, + env, + path::Path, process, }; @@ -53,539 +50,6 @@ enum Command { }, } -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -enum Source { - YouTubeVideo, - YouTubePlaylist, - YouTubeChannel, - X, - Tweet, - TweetThread, - Instagram, - Facebook, - TikTok, - Reddit, - Snapchat, - Local, - Other, -} - -fn expand_shorthand_to_url(path: &str, source: &Source) -> String { - if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) { - if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) { - return format!("https://x.com/i/status/{tweet_id}"); - } - } - - if let Some(path) = path.strip_prefix("instagram:") { - if let Some(id) = path.strip_prefix("reel:") { - return format!("https://www.instagram.com/reel/{id}"); - } - return format!("https://www.instagram.com/{path}"); - } - if let Some(path) = path.strip_prefix("facebook:") { - return format!("https://www.facebook.com/{path}"); - } - if let Some(path) = path.strip_prefix("tiktok:") { - return format!("https://www.tiktok.com/{path}"); - } - if let Some(path) = path.strip_prefix("reddit:") { - return format!("https://www.reddit.com/{path}"); - } - if let Some(path) = path.strip_prefix("snapchat:") { - return format!("https://www.snapchat.com/{path}"); - } - - path.to_string() -} - -// INFO: yt-dlp supports a lot of sites; so, when archiving (for example) a website, the user -// -> should be asked whether they want to archive the whole website or just the video(s) on it. -fn determine_source(path: &str) -> Source { - // INFO: Extractor URLs can be found here: - // -> https://github.com/yt-dlp/yt-dlp/tree/dfc0a84c192a7357dd1768cc345d590253a14fe5/yt_dlp/extractor - // TEST: X posts can have multiple videos. - - // Shorthand schemes: yt: or youtube: - if let Some(after_scheme) = path - .strip_prefix("yt:") - .or_else(|| path.strip_prefix("youtube:")) - { - // video/ID, short/ID, shorts/ID - if after_scheme.starts_with("video/") - || after_scheme.starts_with("short/") - || after_scheme.starts_with("shorts/") - { - return Source::YouTubeVideo; - } - - // playlist/ID - if after_scheme.starts_with("playlist/") { - return Source::YouTubePlaylist; - } - - // channel/ID, c/ID, user/ID, @handle - if after_scheme.starts_with("channel/") - || after_scheme.starts_with("c/") - || after_scheme.starts_with("user/") - || after_scheme.starts_with("@") - { - return Source::YouTubeChannel; - } - } - - // Shorthand schemes: tweet:, x:, or twitter: - if let Some(after_scheme) = path - .strip_prefix("x:") - .or_else(|| path.strip_prefix("twitter:")) - .or_else(|| path.strip_prefix("tweet:")) - { - // For this scope, in comments, N is an alias for a string of type ('twitter' | 'x' | 'tweet'). - - // N:media:id - if after_scheme.starts_with("media:") - && after_scheme - .strip_prefix("media:") - .and_then(parse_tweet_id) - .is_some() - { - return Source::X; - } - - // N:tweet:id or N:x:id - if after_scheme - .strip_prefix("tweet:") - .or_else(|| after_scheme.strip_prefix("x:")) - .and_then(parse_tweet_id) - .is_some() - { - return Source::Tweet; - } - - // N:thread:id - if after_scheme - .strip_prefix("thread:") - .and_then(parse_tweet_id) - .is_some() - { - return Source::TweetThread; - } - - // N:id - if parse_tweet_id(after_scheme).is_some() { - return Source::Tweet; - } - - // N:non-id - return Source::Other; - } - - // Shorthand schemes for other yt-dlp extractors - if path.starts_with("instagram:") { - return Source::Instagram; - } - if path.starts_with("facebook:") { - return Source::Facebook; - } - if path.starts_with("tiktok:") { - return Source::TikTok; - } - if path.starts_with("reddit:") { - return Source::Reddit; - } - if path.starts_with("snapchat:") { - return Source::Snapchat; - } - - if path.starts_with("file://") { - return Source::Local; - } else if path.starts_with("http://") || path.starts_with("https://") { - // Video URLs (watch, youtu.be, shorts) - let video_re = regex::Regex::new(r"^https?://(?:www\.)?(?:youtu\.be/[0-9A-Za-z_-]+|youtube\.com/watch\?v=[0-9A-Za-z_-]+|youtube\.com/shorts/[0-9A-Za-z_-]+)") - .expect("YouTube video URL regex literal must be valid"); - if video_re.is_match(path) { - return Source::YouTubeVideo; - } - - // Playlist URLs - let playlist_re = - regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/playlist\?list=[0-9A-Za-z_-]+") - .expect("YouTube playlist URL regex literal must be valid"); - if playlist_re.is_match(path) { - return Source::YouTubePlaylist; - } - - // Channel or user URLs (channel IDs, /c/, /user/, or @handles) - let channel_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/(?:channel/[0-9A-Za-z_-]+|c/[0-9A-Za-z_-]+|user/[0-9A-Za-z_-]+|@[0-9A-Za-z_-]+)") - .expect("YouTube channel URL regex literal must be valid"); - if channel_re.is_match(path) { - return Source::YouTubeChannel; - } - - if path.starts_with("https://x.com/") { - return Source::X; - } - - if path.starts_with("https://instagram.com/") - || path.starts_with("https://www.instagram.com/") - || path.starts_with("http://instagram.com/") - || path.starts_with("http://www.instagram.com/") - { - return Source::Instagram; - } - - if path.starts_with("https://facebook.com/") - || path.starts_with("https://www.facebook.com/") - || path.starts_with("http://facebook.com/") - || path.starts_with("http://www.facebook.com/") - || path.starts_with("https://fb.watch/") - || path.starts_with("http://fb.watch/") - { - return Source::Facebook; - } - - if path.starts_with("https://tiktok.com/") - || path.starts_with("https://www.tiktok.com/") - || path.starts_with("http://tiktok.com/") - || path.starts_with("http://www.tiktok.com/") - { - return Source::TikTok; - } - - if path.starts_with("https://reddit.com/") - || path.starts_with("https://www.reddit.com/") - || path.starts_with("http://reddit.com/") - || path.starts_with("http://www.reddit.com/") - || path.starts_with("https://redd.it/") - || path.starts_with("http://redd.it/") - { - return Source::Reddit; - } - - if path.starts_with("https://snapchat.com/") - || path.starts_with("https://www.snapchat.com/") - || path.starts_with("http://snapchat.com/") - || path.starts_with("http://www.snapchat.com/") - { - return Source::Snapchat; - } - } - if Path::new(path).exists() { - return Source::Local; - } - Source::Other -} - -fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result { - let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); - - println!("Checking {}", path.display()); - - Ok(path.exists()) -} - -fn move_temp_to_raw(file: &Path, hash: &str, store_path: &Path) -> Result<()> { - let file_extension = file - .extension() - .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())); - let raw_relpath = raw_relative_path_from_hash(hash, &file_extension)?; - let destination = store_path.join(raw_relpath); - - if let Some(parent) = destination.parent() { - fs::create_dir_all(parent)?; - } - - fs::rename(file, destination)?; - - Ok(()) -} - -fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result { - let mut chars = hash.chars(); - let first_letter = chars.next().context("hash must not be empty")?; - let second_letter = chars - .next() - .context("hash must be at least two characters")?; - - Ok(PathBuf::from("raw") - .join(first_letter.to_string()) - .join(second_letter.to_string()) - .join(format!("{hash}{file_extension}"))) -} - -fn path_to_store_string(path: &Path) -> String { - path.to_string_lossy().replace('\\', "/") -} - -fn extension_without_dot(file_extension: &str) -> Option { - file_extension - .strip_prefix('.') - .filter(|extension| !extension.is_empty()) - .map(|extension| extension.to_string()) -} - -fn blob_record_for_raw_relpath( - store_path: &Path, - raw_relpath: &Path, -) -> Result { - let absolute_path = store_path.join(raw_relpath); - let file_name = raw_relpath - .file_name() - .and_then(|name| name.to_str()) - .context("raw artifact path must have a UTF-8 file name")?; - let (sha256, extension) = match file_name.rsplit_once('.') { - Some((hash, extension)) => (hash.to_string(), Some(extension.to_string())), - None => (file_name.to_string(), None), - }; - - Ok(database::BlobRecord { - sha256, - byte_size: fs::metadata(&absolute_path) - .with_context(|| format!("failed to stat raw artifact {}", absolute_path.display()))? - .len() as i64, - mime_type: None, - extension, - raw_relpath: path_to_store_string(raw_relpath), - }) -} - -fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str) { - match source { - Source::YouTubeVideo => ("youtube", "video", "video"), - Source::YouTubePlaylist => ("youtube", "playlist", "container"), - Source::YouTubeChannel => ("youtube", "channel", "container"), - Source::X => ("x", "post", "video"), - Source::Tweet => ("x", "tweet", "tweet_json"), - Source::TweetThread => ("x", "tweet_thread", "tweet_json"), - Source::Instagram => ("instagram", "post", "video"), - Source::Facebook => ("facebook", "post", "video"), - Source::TikTok => ("tiktok", "video", "video"), - Source::Reddit => ("reddit", "post", "video"), - Source::Snapchat => ("snapchat", "story", "video"), - Source::Local => ("local", "file", "file"), - Source::Other => ("other", "unknown", "unknown"), - } -} - -fn local_file_extension(path: &str) -> String { - Path::new(path.trim_start_matches("file://")) - .extension() - .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) -} - -fn media_file_extension(source: Source, path: &str) -> String { - match source { - Source::YouTubeVideo - | Source::X - | Source::Instagram - | Source::Facebook - | Source::TikTok - | Source::Reddit - | Source::Snapchat => ".mp4".to_string(), - Source::Local => local_file_extension(path), - _ => String::new(), - } -} - -fn tweet_id_from_archive_path(path: &str) -> Option { - path.split(':').next_back().and_then(parse_tweet_id) -} - -fn create_structured_root(store_path: &Path, entry: &database::ArchivedEntry) -> Result<()> { - debug_assert!(entry.entry_uid.starts_with("entry_")); - fs::create_dir_all(store_path.join(&entry.structured_root_relpath))?; - Ok(()) -} - -fn record_media_entry( - conn: &rusqlite::Connection, - store_path: &Path, - user_id: i64, - run: &database::ArchiveRun, - item: &database::ArchiveRunItem, - requested_locator: &str, - canonical_locator: &str, - source: Source, - hash: &str, - file_extension: &str, - byte_size: i64, -) -> Result { - debug_assert!(run.run_uid.starts_with("run_")); - debug_assert!(item.item_uid.starts_with("item_")); - let (source_kind, entity_kind, representation_kind) = source_metadata(source); - let raw_relpath = raw_relative_path_from_hash(hash, file_extension)?; - let blob = database::BlobRecord { - sha256: hash.to_string(), - byte_size, - mime_type: None, - extension: extension_without_dot(file_extension), - raw_relpath: path_to_store_string(&raw_relpath), - }; - let blob_id = database::upsert_blob(conn, &blob)?; - let source_identity_id = database::upsert_source_identity( - conn, - source_kind, - entity_kind, - None, - Some(canonical_locator), - canonical_locator, - )?; - let entry = database::create_archived_entry( - conn, - &database::NewEntry { - source_identity_id, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: source_kind.to_string(), - entity_kind: entity_kind.to_string(), - title: None, - visibility: "private".to_string(), - representation_kind: representation_kind.to_string(), - source_metadata_json: json!({ - "requested_locator": requested_locator, - "canonical_locator": canonical_locator - }) - .to_string(), - display_metadata_json: None, - }, - )?; - create_structured_root(store_path, &entry)?; - database::add_entry_artifact( - conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: "primary_media".to_string(), - storage_area: "raw".to_string(), - relpath: blob.raw_relpath, - blob_id: Some(blob_id), - logical_path: None, - metadata_json: None, - }, - )?; - database::complete_archive_run_item(conn, item.id, entry.id)?; - Ok(entry) -} - -fn record_tweet_entry( - conn: &rusqlite::Connection, - store_path: &Path, - user_id: i64, - run: &database::ArchiveRun, - item: &database::ArchiveRunItem, - requested_locator: &str, - source: Source, - tweet_id: &str, -) -> Result { - debug_assert!(run.run_uid.starts_with("run_")); - debug_assert!(item.item_uid.starts_with("item_")); - let (source_kind, entity_kind, representation_kind) = source_metadata(source); - let canonical_locator = format!("https://x.com/i/status/{tweet_id}"); - let source_identity_id = database::upsert_source_identity( - conn, - source_kind, - entity_kind, - Some(tweet_id), - Some(&canonical_locator), - &canonical_locator, - )?; - let entry = database::create_archived_entry( - conn, - &database::NewEntry { - source_identity_id, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: source_kind.to_string(), - entity_kind: entity_kind.to_string(), - title: None, - visibility: "private".to_string(), - representation_kind: representation_kind.to_string(), - source_metadata_json: json!({ - "tweet_id": tweet_id, - "requested_locator": requested_locator - }) - .to_string(), - display_metadata_json: None, - }, - )?; - create_structured_root(store_path, &entry)?; - - let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); - database::add_entry_artifact( - conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: "raw_tweet_json".to_string(), - storage_area: "raw_tweets".to_string(), - relpath: path_to_store_string(&tweet_json_relpath), - blob_id: None, - logical_path: None, - metadata_json: None, - }, - )?; - - let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; - for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? { - let raw_path = PathBuf::from(&raw_relpath); - let blob = blob_record_for_raw_relpath(store_path, &raw_path)?; - let blob_id = database::upsert_blob(conn, &blob)?; - database::add_entry_artifact( - conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: role, - storage_area: "raw".to_string(), - relpath: raw_relpath, - blob_id: Some(blob_id), - logical_path: None, - metadata_json: None, - }, - )?; - } - - database::complete_archive_run_item(conn, item.id, entry.id)?; - Ok(entry) -} - -fn tweet_raw_artifacts(tweet_json: &str) -> Result> { - let regex = regex::Regex::new(r#""(avatar_local_path|local_path)": "([^"\n]+)""#)?; - let mut seen = HashSet::new(); - let mut artifacts = Vec::new(); - - for captures in regex.captures_iter(tweet_json) { - let relpath = captures[2].to_string(); - if !relpath.starts_with("raw/") || !seen.insert(relpath.clone()) { - continue; - } - - let role = if &captures[1] == "avatar_local_path" { - "avatar" - } else { - "media" - }; - artifacts.push((role.to_string(), relpath)); - } - - Ok(artifacts) -} - -fn fail_archive_and_exit( - conn: &rusqlite::Connection, - run: &database::ArchiveRun, - item: &database::ArchiveRunItem, - message: &str, -) -> ! { - let _ = database::fail_archive_run_item(conn, item.id, message); - let _ = database::fail_archive_run(conn, run.id, message); - eprintln!("{message}"); - process::exit(1); -} - fn main() -> Result<()> { let args = Args::parse(); @@ -598,214 +62,9 @@ fn main() -> Result<()> { process::exit(1); } }; - - // let download_id = uuid::Uuid::new_v4(); - let timestamp = Local::now().format("%Y-%m-%dT%H-%M-%S%.3f").to_string(); - - let store_path_string_file = archive_path.join("store_path"); - let store_path = match fs::read_to_string(store_path_string_file) { - Ok(p) => PathBuf::from(p.trim()), - Err(e) => { - eprintln!("Failed to read store path: {e}"); - process::exit(1); - } - }; - - let source = determine_source(path); - let (source_kind, entity_kind, _) = source_metadata(source); - let conn = database::open_or_initialize(&archive_path)?; - let user_id = database::ensure_default_user(&conn)?; - let run = database::create_archive_run(&conn, user_id, 1)?; - let item = database::create_archive_run_item( - &conn, - run.id, - None, - 0, - path, - None, - source_kind, - entity_kind, - )?; - - // Sources: Tweets or Twitter Threads - match source { - Source::Other => { - fail_archive_and_exit( - &conn, - &run, - &item, - "Archiving from this source is not yet implemented.", - ); - } - Source::Tweet | Source::TweetThread => { - let tweet_id = match tweet_id_from_archive_path(path) { - Some(tweet_id) => tweet_id, - None => fail_archive_and_exit( - &conn, - &run, - &item, - "Failed to archive tweet: invalid tweet ID", - ), - }; - - match downloader::tweets::archive( - path, - source == Source::TweetThread, - &store_path, - ×tamp, - ) { - Ok(true) => { - record_tweet_entry( - &conn, - &store_path, - user_id, - &run, - &item, - path, - source, - &tweet_id, - )?; - database::finish_archive_run(&conn, run.id)?; - println!( - "Tweet archived successfully to {}", - store_path.join("raw_tweets").display() - ); - return Ok(()); - } - Ok(false) => { - record_tweet_entry( - &conn, - &store_path, - user_id, - &run, - &item, - path, - source, - &tweet_id, - )?; - database::finish_archive_run(&conn, run.id)?; - println!( - "Tweet already archived in {}", - store_path.join("raw_tweets").display() - ); - return Ok(()); - } - Err(e) => { - fail_archive_and_exit( - &conn, - &run, - &item, - &format!("Failed to archive tweet: {e}"), - ); - } - } - } - _ => {} - } - - // Sources, for which yt-dlp is needed - let requested_path = path.to_string(); - let path = expand_shorthand_to_url(path, &source); - let hash = match source { - Source::YouTubeVideo - | Source::X - | Source::Instagram - | Source::Facebook - | Source::TikTok - | Source::Reddit - | Source::Snapchat => { - match downloader::ytdlp::download(path.clone(), &store_path, ×tamp) { - Ok(h) => h, - Err(e) => { - fail_archive_and_exit( - &conn, - &run, - &item, - &format!("Failed to download media: {e}"), - ); - } - } - } - Source::Local => { - match downloader::local::save(path.clone(), &store_path, ×tamp) { - Ok(h) => h, - Err(e) => { - fail_archive_and_exit( - &conn, - &run, - &item, - &format!("Failed to archive local file: {e}"), - ); - } - } - } - Source::YouTubePlaylist | Source::YouTubeChannel => { - fail_archive_and_exit( - &conn, - &run, - &item, - "Playlist and channel container expansion are not yet implemented.", - ); - } - _ => unreachable!(), - }; - - let file_extension = media_file_extension(source, &path); - let temp_file = store_path - .join("temp") - .join(×tamp) - .join(format!("{timestamp}{file_extension}")); - let byte_size = fs::metadata(&temp_file) - .with_context(|| format!("failed to stat staged file {}", temp_file.display()))? - .len() as i64; - - let hash_exists = hash_exists(&hash, &file_extension, &store_path)?; - - // TODO: check for repeated archives? - // There could be one of the following: - // - We are literally archiving the same path over again. - // - We are archiving a different path, which had this file. E.g.: we archived a - // website before which had this YouTube video, and while recursively archiving - // everything, we also archived the YouTube video although it wasn't our main - // target. This means that we should archive again; whereas with the first case... - // Not sure. Need to think about this. - // ---- - // Thinking about it a day later... - // If we are specifically archiving a YouTube video, it could also be two of the - // above. So yeah, just create a new DB entry and symlink the Raw to the Structured - // Dir or whatever. it's midnight and my brain ain't wording/braining. - if hash_exists { - println!("File already archived."); - let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); - } else { - move_temp_to_raw( - &store_path - .join("temp") - .join(×tamp) - .join(format!("{timestamp}{file_extension}")), - &hash, - &store_path, - )?; - let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); - - println!("File archived successfully."); - } - - record_media_entry( - &conn, - &store_path, - user_id, - &run, - &item, - &requested_path, - &path, - source, - &hash, - &file_extension, - byte_size, - )?; - database::finish_archive_run(&conn, run.id)?; - + let archive_paths = archive::read_archive_paths(&archive_path)?; + let result = archivr_core::capture::perform_capture(&archive_paths, path)?; + println!("Archived: run {}", result.run_uid); Ok(()) } @@ -841,481 +100,3 @@ fn main() -> Result<()> { } } -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - struct TestCase<'a> { - url: &'a str, - expected: Source, - } - - #[test] - fn test_tweet_sources() { - let cases = [ - TestCase { - url: "tweet:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "x:tweet:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "x:x:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "twitter:x:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "twitter:tweet:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "tweet:media:1234567890", - expected: Source::X, - }, - TestCase { - url: "x:media:1234567890", - expected: Source::X, - }, - TestCase { - url: "x:thread:1234567890", - expected: Source::TweetThread, - }, - TestCase { - url: "twitter:thread:1234567890", - expected: Source::TweetThread, - }, - TestCase { - url: "tweet:thread:1234567890", - expected: Source::TweetThread, - }, - TestCase { - url: "tweet:not-a-number", - expected: Source::Other, - }, - TestCase { - url: "tweet:media:not-a-number", - expected: Source::Other, - }, - TestCase { - url: "x:media:not-a-number", - expected: Source::Other, - }, - ]; - - for case in &cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_resolve_source_path() { - assert_eq!( - expand_shorthand_to_url("tweet:media:1234567890", &Source::X), - "https://x.com/i/status/1234567890" - ); - assert_eq!( - expand_shorthand_to_url("instagram:reel/ABC123", &Source::Instagram), - "https://www.instagram.com/reel/ABC123" - ); - assert_eq!( - expand_shorthand_to_url("facebook:watch?v=123456", &Source::Facebook), - "https://www.facebook.com/watch?v=123456" - ); - assert_eq!( - expand_shorthand_to_url("tiktok:@someone/video/123456789", &Source::TikTok), - "https://www.tiktok.com/@someone/video/123456789" - ); - assert_eq!( - expand_shorthand_to_url("reddit:r/videos/comments/abc123/example", &Source::Reddit), - "https://www.reddit.com/r/videos/comments/abc123/example" - ); - assert_eq!( - expand_shorthand_to_url("snapchat:discover/some-story/1234567890", &Source::Snapchat), - "https://www.snapchat.com/discover/some-story/1234567890" - ); - assert_eq!( - expand_shorthand_to_url("tweet:1234567890", &Source::Tweet), - "tweet:1234567890" - ); - } - - #[test] - fn test_youtube_sources() { - // --- YouTube Video URLs --- - let video_cases = [ - TestCase { - url: "https://www.youtube.com/watch?v=UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "https://youtu.be/UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "https://www.youtube.com/shorts/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - ]; - - for case in &video_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - - // --- YouTube Playlist URLs --- - let playlist_cases = [TestCase { - url: "https://www.youtube.com/playlist?list=PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", - expected: Source::YouTubePlaylist, - }]; - - for case in &playlist_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - - // --- YouTube Channel URLs --- - let channel_cases = [ - TestCase { - url: "https://www.youtube.com/channel/CoreDumpped", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://www.youtube.com/@CoreDumpped", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://www.youtube.com/c/YouTubeCreators", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://www.youtube.com/user/pewdiepie", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://youtube.com/@pewdiepie?si=KOcLN_KPYNpe5f_8", - expected: Source::YouTubeChannel, - }, - ]; - - for case in &channel_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - - // --- Shorthand scheme URLs --- - let shorthand_cases = [ - // Videos - TestCase { - url: "yt:video/UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "youtube:video/UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "yt:short/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "yt:shorts/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "youtube:shorts/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - // Playlists - TestCase { - url: "yt:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", - expected: Source::YouTubePlaylist, - }, - TestCase { - url: "youtube:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", - expected: Source::YouTubePlaylist, - }, - // Channels - TestCase { - url: "yt:channel/UCxyz123", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "yt:c/YouTubeCreators", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "yt:user/pewdiepie", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "youtube:@CoreDumpped", - expected: Source::YouTubeChannel, - }, - ]; - - for case in &shorthand_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_x_sources() { - let x_cases = [ - TestCase { - url: "https://x.com/some_post", - expected: Source::X, - }, - TestCase { - url: "x:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "twitter:1234567890", - expected: Source::Tweet, - }, - ]; - - for case in &x_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_other_social_sources() { - let social_cases = [ - TestCase { - url: "https://www.instagram.com/reel/ABC123/", - expected: Source::Instagram, - }, - TestCase { - url: "instagram:reel/ABC123", - expected: Source::Instagram, - }, - TestCase { - url: "https://www.facebook.com/watch/?v=123456", - expected: Source::Facebook, - }, - TestCase { - url: "facebook:watch?v=123456", - expected: Source::Facebook, - }, - TestCase { - url: "https://www.tiktok.com/@someone/video/123456789", - expected: Source::TikTok, - }, - TestCase { - url: "tiktok:@someone/video/123456789", - expected: Source::TikTok, - }, - TestCase { - url: "https://www.reddit.com/r/videos/comments/abc123/example/", - expected: Source::Reddit, - }, - TestCase { - url: "reddit:r/videos/comments/abc123/example", - expected: Source::Reddit, - }, - TestCase { - url: "https://www.snapchat.com/discover/some-story/1234567890", - expected: Source::Snapchat, - }, - TestCase { - url: "snapchat:discover/some-story/1234567890", - expected: Source::Snapchat, - }, - ]; - - for case in &social_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_non_youtube_sources() { - let other_cases = [ - TestCase { - url: "file:///local/path/file.mp4", - expected: Source::Local, - }, - TestCase { - url: "https://example.com/", - expected: Source::Other, - }, - TestCase { - url: "https://example.com/?redirect=instagram.com/reel/ABC123", - expected: Source::Other, - }, - TestCase { - url: "https://notfacebook.com/watch?v=123456", - expected: Source::Other, - }, - ]; - - for case in &other_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_existing_local_path_source() { - let path = env::current_dir().unwrap().join("Cargo.toml"); - assert_eq!( - determine_source(path.to_str().unwrap()), - Source::Local, - "existing filesystem paths should be archived as local files" - ); - } - - #[test] - fn test_initialize_store_directories() { - let store_path = env::temp_dir().join(format!( - "archivr-test-{}", - Local::now().format("%Y%m%d%H%M%S%3f") - )); - - archive::initialize_store_directories(&store_path).unwrap(); - - assert!(store_path.join("raw").is_dir()); - assert!(store_path.join("raw_tweets").is_dir()); - assert!(store_path.join("structured").is_dir()); - assert!(store_path.join("temp").is_dir()); - assert!(!store_path.join("tmp").exists()); - - fs::remove_dir_all(store_path).unwrap(); - } - - #[test] - fn test_record_tweet_entry_links_json_and_raw_artifacts() { - let store_path = env::temp_dir().join(format!( - "archivr-tweet-db-test-{}", - Local::now().format("%Y%m%d%H%M%S%3f") - )); - let _ = fs::remove_dir_all(&store_path); - archive::initialize_store_directories(&store_path).unwrap(); - fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap(); - fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap(); - fs::write( - store_path - .join("raw") - .join("a") - .join("b") - .join("abcdef.jpg"), - b"avatar", - ) - .unwrap(); - fs::write( - store_path - .join("raw") - .join("c") - .join("d") - .join("cdef01.mp4"), - b"media", - ) - .unwrap(); - fs::write( - store_path.join("raw_tweets").join("tweet-123.json"), - r#"{ - "author": { "avatar_local_path": "raw/a/b/abcdef.jpg" }, - "entities": { "media": [{ "local_path": "raw/c/d/cdef01.mp4" }] } -}"#, - ) - .unwrap(); - - let conn = rusqlite::Connection::open_in_memory().unwrap(); - database::initialize_schema(&conn).unwrap(); - let user_id = database::ensure_default_user(&conn).unwrap(); - let run = database::create_archive_run(&conn, user_id, 1).unwrap(); - let item = database::create_archive_run_item( - &conn, - run.id, - None, - 0, - "tweet:123", - None, - "x", - "tweet", - ) - .unwrap(); - - let entry = record_tweet_entry( - &conn, - &store_path, - user_id, - &run, - &item, - "tweet:123", - Source::Tweet, - "123", - ) - .unwrap(); - database::finish_archive_run(&conn, run.id).unwrap(); - - let artifact_count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM entry_artifacts WHERE entry_id = ?1", - [entry.id], - |row| row.get(0), - ) - .unwrap(); - let blob_count: i64 = conn - .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get(0)) - .unwrap(); - let run_status: String = conn - .query_row( - "SELECT status FROM archive_runs WHERE id = ?1", - [run.id], - |row| row.get(0), - ) - .unwrap(); - - assert_eq!(artifact_count, 3); - assert_eq!(blob_count, 2); - assert_eq!(run_status, "completed"); - assert!(store_path.join(&entry.structured_root_relpath).is_dir()); - - let _ = fs::remove_dir_all(store_path); - } -} diff --git a/crates/archivr-core/src/archive.rs b/crates/archivr-core/src/archive.rs index 0d507c0..be14c24 100644 --- a/crates/archivr-core/src/archive.rs +++ b/crates/archivr-core/src/archive.rs @@ -25,6 +25,7 @@ pub struct EntrySummary { pub original_url: Option, pub artifact_count: i64, pub total_artifact_bytes: i64, + pub parent_entry_uid: Option, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] @@ -57,6 +58,20 @@ pub struct RunSummary { pub error_summary: Option, } +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct Tag { + pub tag_uid: String, + pub name: String, + pub slug: String, + pub full_path: String, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct TagNode { + pub tag: Tag, + pub children: Vec, +} + pub fn find_archive_path_from(start: &Path) -> Result> { let mut dir = start.to_path_buf(); loop { @@ -167,7 +182,8 @@ pub fn list_root_entries(conn: &rusqlite::Connection) -> Result Result>>()?; @@ -278,6 +295,384 @@ pub fn list_runs(conn: &rusqlite::Connection) -> Result> { Ok(runs) } +/// Resolves an artifact to its absolute on-disk path under `store_path`. +/// +/// `artifact.relpath` is a store-relative path (e.g. `raw/a/b/abc.pdf`). +/// The returned path is canonicalized. Returns an error if the resolved path +/// escapes `store_path` (path traversal protection) or if the file does not exist. +pub fn resolve_artifact_path( + store_path: &Path, + artifact: &EntryArtifactSummary, +) -> Result { + let joined = store_path.join(&artifact.relpath); + let canonical_store = store_path + .canonicalize() + .with_context(|| format!("failed to canonicalize store path: {}", store_path.display()))?; + let canonical_artifact = joined + .canonicalize() + .with_context(|| format!("artifact path does not exist: {}", joined.display()))?; + if !canonical_artifact.starts_with(&canonical_store) { + bail!( + "artifact path escapes store: {}", + canonical_artifact.display() + ); + } + Ok(canonical_artifact) +} + +#[derive(Debug, Clone, Default)] +pub struct SearchEntriesQuery { + /// Free-text term: LIKE-matched against title, canonical_url, entry_uid, source_kind, entity_kind, visibility + pub q: Option, + /// Exact match on e.source_kind + pub source_kind: Option, + /// Exact match on e.entity_kind + pub entity_kind: Option, + /// LIKE-matched against si.canonical_url + pub url: Option, + /// LIKE-matched against e.title + pub title: Option, + /// e.archived_at >= after (inclusive, ISO 8601) + pub after: Option, + /// e.archived_at < before (exclusive, ISO 8601) + pub before: Option, + /// Tag full_path filter; includes all entries (root + child) matching the tag subtree + pub tag: Option, +} + +/// Parses a raw search string into a [`SearchEntriesQuery`]. +/// +/// Recognized prefixes: `source:`, `type:`, `url:`, `title:`, `after:`, `before:`, `tag:`. +/// Tokens with an unrecognized `prefix:` return `Err(prefix)`. +/// Remaining non-prefix tokens are joined as the free-text `q`. +/// Quoted values (`title:"resume templates"`) are supported for single-word values +/// after the colon; leading/trailing double quotes are stripped. +pub fn parse_search_query(raw: &str) -> Result { + let mut query = SearchEntriesQuery::default(); + let mut free_text_tokens: Vec<&str> = Vec::new(); + + for token in raw.split_whitespace() { + if let Some(colon_pos) = token.find(':') { + let prefix = &token[..colon_pos]; + let value_raw = &token[colon_pos + 1..]; + // Strip surrounding double quotes if present + let value = value_raw.trim_matches('"').to_string(); + + match prefix { + "source" => query.source_kind = Some(value), + "type" => query.entity_kind = Some(value), + "url" => query.url = Some(value), + "title" => query.title = Some(value), + "after" => query.after = Some(value), + "before" => query.before = Some(value), + "tag" => query.tag = Some(value), + other => return Err(other.to_string()), + } + } else { + free_text_tokens.push(token); + } + } + + let q = free_text_tokens.join(" "); + query.q = if q.is_empty() { None } else { Some(q) }; + + Ok(query) +} + +const ENTRY_SELECT_COLS: &str = + "SELECT e.entry_uid, e.archived_at, e.source_kind, e.entity_kind, e.title, \ + e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, \ + COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \ + parent.entry_uid AS parent_entry_uid"; + +const ENTRY_FROM_JOINS: &str = + "FROM archived_entries e \ + JOIN source_identities si ON si.id = e.source_identity_id \ + LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id \ + LEFT JOIN blobs b ON b.id = ea.blob_id \ + LEFT JOIN archived_entries parent ON parent.id = e.parent_entry_id"; + +/// Searches archived entries matching all non-`None` fields in `query`. +/// +/// Without a tag filter, returns root entries only (same scope as [`list_root_entries`]). +/// With a tag filter, returns ALL entries (root and child) assigned to that tag subtree. +pub fn search_entries( + conn: &rusqlite::Connection, + query: &SearchEntriesQuery, +) -> Result> { + let mut params: Vec = Vec::new(); + let mut sql; + + if let Some(tag_path) = &query.tag { + params.push(tag_path.clone()); + sql = format!( + "WITH RECURSIVE descendants(id) AS (\ + SELECT id FROM tags WHERE full_path = ?1 \ + UNION ALL \ + SELECT child.id FROM tags child \ + JOIN descendants d ON child.parent_tag_id = d.id\ + ) {} {} \ + JOIN entry_tag_assignments eta ON eta.entry_id = e.id \ + JOIN descendants d ON eta.tag_id = d.id \ + WHERE 1=1", + ENTRY_SELECT_COLS, + ENTRY_FROM_JOINS, + ); + } else { + sql = format!( + "{} {} WHERE e.parent_entry_id IS NULL", + ENTRY_SELECT_COLS, + ENTRY_FROM_JOINS, + ); + } + + if let Some(q) = &query.q { + let term = format!("%{}%", q.to_lowercase()); + let n = params.len() + 1; + sql.push_str(&format!( + " AND (LOWER(e.title) LIKE ?{n} OR LOWER(si.canonical_url) LIKE ?{n} \ + OR LOWER(e.entry_uid) LIKE ?{n} OR LOWER(e.source_kind) LIKE ?{n} \ + OR LOWER(e.entity_kind) LIKE ?{n} OR LOWER(e.visibility) LIKE ?{n})" + )); + params.push(term); + } + if let Some(sk) = &query.source_kind { + let n = params.len() + 1; + sql.push_str(&format!(" AND e.source_kind = ?{n}")); + params.push(sk.clone()); + } + if let Some(ek) = &query.entity_kind { + let n = params.len() + 1; + sql.push_str(&format!(" AND e.entity_kind = ?{n}")); + params.push(ek.clone()); + } + if let Some(u) = &query.url { + let n = params.len() + 1; + sql.push_str(&format!(" AND LOWER(si.canonical_url) LIKE ?{n}")); + params.push(format!("%{}%", u.to_lowercase())); + } + if let Some(t) = &query.title { + let n = params.len() + 1; + sql.push_str(&format!(" AND LOWER(e.title) LIKE ?{n}")); + params.push(format!("%{}%", t.to_lowercase())); + } + if let Some(a) = &query.after { + let n = params.len() + 1; + sql.push_str(&format!(" AND e.archived_at >= ?{n}")); + params.push(a.clone()); + } + if let Some(b) = &query.before { + let n = params.len() + 1; + sql.push_str(&format!(" AND e.archived_at < ?{n}")); + params.push(b.clone()); + } + + sql.push_str(" GROUP BY e.id ORDER BY e.archived_at DESC, e.id DESC"); + + let mut stmt = conn.prepare(&sql)?; + let entries = stmt + .query_map(rusqlite::params_from_iter(params.iter()), |row| { + Ok(EntrySummary { + entry_uid: row.get(0)?, + archived_at: row.get(1)?, + source_kind: row.get(2)?, + entity_kind: row.get(3)?, + title: row.get(4)?, + visibility: row.get(5)?, + original_url: row.get(6)?, + artifact_count: row.get(7)?, + total_artifact_bytes: row.get(8)?, + parent_entry_uid: row.get(9)?, + }) + })? + .collect::>>()?; + Ok(entries) +} + +fn tag_by_id(conn: &rusqlite::Connection, id: i64) -> Result { + conn.query_row( + "SELECT tag_uid, name, slug, full_path FROM tags WHERE id = ?1", + [id], + |row| { + Ok(Tag { + tag_uid: row.get(0)?, + name: row.get(1)?, + slug: row.get(2)?, + full_path: row.get(3)?, + }) + }, + ) + .context("tag not found by id") +} + +/// Creates all tag path segments (idempotent) and returns the leaf `Tag`. +pub fn create_tag(conn: &rusqlite::Connection, full_path: &str) -> Result { + let id = database::create_tag_path(conn, full_path)?; + tag_by_id(conn, id) +} + +/// Returns the full tag tree with root nodes at the top level and children nested. +pub fn list_tag_tree(conn: &rusqlite::Connection) -> Result> { + use std::collections::HashMap; + + let records = database::list_all_tags(conn)?; + + let mut by_parent: HashMap, Vec> = HashMap::new(); + for record in records { + by_parent.entry(record.parent_tag_id).or_default().push(record); + } + + fn build_nodes( + parent_id: Option, + by_parent: &HashMap, Vec>, + ) -> Vec { + let Some(children) = by_parent.get(&parent_id) else { + return Vec::new(); + }; + children + .iter() + .map(|r| TagNode { + tag: Tag { + tag_uid: r.tag_uid.clone(), + name: r.name.clone(), + slug: r.slug.clone(), + full_path: r.full_path.clone(), + }, + children: build_nodes(Some(r.id), by_parent), + }) + .collect() + } + + Ok(build_nodes(None, &by_parent)) +} + +/// Returns the tags assigned to an entry. +/// +/// Returns `Ok(None)` if the entry_uid does not exist (caller maps to 404). +/// Returns `Ok(Some([]))` if the entry exists but has no tags. +pub fn get_entry_tags( + conn: &rusqlite::Connection, + entry_uid: &str, +) -> Result>> { + let Some(entry_id) = conn + .query_row( + "SELECT id FROM archived_entries WHERE entry_uid = ?1", + [entry_uid], + |row| row.get::<_, i64>(0), + ) + .optional()? + else { + return Ok(None); + }; + let records = database::list_tags_for_entry(conn, entry_id)?; + Ok(Some( + records + .into_iter() + .map(|r| Tag { + tag_uid: r.tag_uid, + name: r.name, + slug: r.slug, + full_path: r.full_path, + }) + .collect(), + )) +} + +/// Assigns a tag (by full path, creating it if needed) to an entry. +/// +/// Returns `Ok(None)` if the entry_uid does not exist. +/// Returns `Ok(Some(tag))` on success. +pub fn assign_entry_tag( + conn: &rusqlite::Connection, + entry_uid: &str, + tag_full_path: &str, +) -> Result> { + let Some(entry_id) = conn + .query_row( + "SELECT id FROM archived_entries WHERE entry_uid = ?1", + [entry_uid], + |row| row.get::<_, i64>(0), + ) + .optional()? + else { + return Ok(None); + }; + let tag_id = database::create_tag_path(conn, tag_full_path)?; + database::assign_entry_to_tag(conn, entry_id, tag_id)?; + Ok(Some(tag_by_id(conn, tag_id)?)) +} + +/// Removes a tag assignment from an entry. +/// +/// Returns `Ok(false)` if either the entry_uid or tag_uid is not found. +/// Returns `Ok(true)` on success (even if no row was deleted, i.e. assignment didn't exist). +pub fn remove_entry_tag( + conn: &rusqlite::Connection, + entry_uid: &str, + tag_uid: &str, +) -> Result { + let Some(entry_id) = conn + .query_row( + "SELECT id FROM archived_entries WHERE entry_uid = ?1", + [entry_uid], + |row| row.get::<_, i64>(0), + ) + .optional()? + else { + return Ok(false); + }; + let Some(tag_record) = database::get_tag_by_uid(conn, tag_uid)? else { + return Ok(false); + }; + database::remove_entry_tag_assignment(conn, entry_id, tag_record.id)?; + Ok(true) +} + +/// Returns all entries (root and child) assigned to any tag in the subtree rooted at `tag_full_path`. +pub fn entries_for_tag( + conn: &rusqlite::Connection, + tag_full_path: &str, +) -> Result> { + let mut stmt = conn.prepare( + "WITH RECURSIVE descendants(id) AS ( + SELECT id FROM tags WHERE full_path = ?1 + UNION ALL + SELECT child.id FROM tags child + JOIN descendants d ON child.parent_tag_id = d.id + ) + SELECT e.entry_uid, e.archived_at, e.source_kind, e.entity_kind, e.title, + e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, + COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, + parent.entry_uid AS parent_entry_uid + FROM archived_entries e + JOIN source_identities si ON si.id = e.source_identity_id + LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id + LEFT JOIN blobs b ON b.id = ea.blob_id + LEFT JOIN archived_entries parent ON parent.id = e.parent_entry_id + JOIN entry_tag_assignments eta ON eta.entry_id = e.id + JOIN descendants d ON eta.tag_id = d.id + GROUP BY e.id + ORDER BY e.archived_at DESC, e.id DESC", + )?; + let entries = stmt + .query_map([tag_full_path], |row| { + Ok(EntrySummary { + entry_uid: row.get(0)?, + archived_at: row.get(1)?, + source_kind: row.get(2)?, + entity_kind: row.get(3)?, + title: row.get(4)?, + visibility: row.get(5)?, + original_url: row.get(6)?, + artifact_count: row.get(7)?, + total_artifact_bytes: row.get(8)?, + parent_entry_uid: row.get(9)?, + }) + })? + .collect::>>()?; + Ok(entries) +} + #[cfg(test)] mod tests { use super::*; @@ -435,4 +830,367 @@ mod tests { assert_eq!(runs.len(), 1); assert_eq!(runs[0].status, "completed"); } + + #[test] + fn resolve_artifact_path_returns_absolute_path_within_store() { + let root = unique_path("archivr-resolve-artifact"); + let store_path = root.join("store"); + fs::create_dir_all(store_path.join("raw/a/b")).unwrap(); + let artifact_file = store_path.join("raw/a/b/abc.pdf"); + fs::write(&artifact_file, b"data").unwrap(); + + let artifact = EntryArtifactSummary { + artifact_role: "primary".to_string(), + storage_area: "raw".to_string(), + relpath: "raw/a/b/abc.pdf".to_string(), + byte_size: Some(4), + }; + let resolved = resolve_artifact_path(&store_path, &artifact).unwrap(); + assert_eq!(resolved, artifact_file.canonicalize().unwrap()); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn resolve_artifact_path_rejects_traversal() { + let root = unique_path("archivr-resolve-traversal"); + let store_path = root.join("store"); + fs::create_dir_all(&store_path).unwrap(); + let artifact = EntryArtifactSummary { + artifact_role: "primary".to_string(), + storage_area: "raw".to_string(), + relpath: "../escaped.txt".to_string(), + byte_size: None, + }; + assert!(resolve_artifact_path(&store_path, &artifact).is_err()); + let _ = fs::remove_dir_all(&root); + } + + // ---- parse_search_query tests ---- + + #[test] + fn parse_empty_query_returns_default() { + let q = parse_search_query("").unwrap(); + assert!(q.q.is_none()); + assert!(q.source_kind.is_none()); + assert!(q.entity_kind.is_none()); + } + + #[test] + fn parse_plain_text_sets_q() { + let q = parse_search_query("polymarket").unwrap(); + assert_eq!(q.q.as_deref(), Some("polymarket")); + } + + #[test] + fn parse_prefix_source_sets_source_kind() { + let q = parse_search_query("source:x").unwrap(); + assert_eq!(q.source_kind.as_deref(), Some("x")); + assert!(q.q.is_none()); + } + + #[test] + fn parse_prefix_type_sets_entity_kind() { + let q = parse_search_query("type:tweet").unwrap(); + assert_eq!(q.entity_kind.as_deref(), Some("tweet")); + } + + #[test] + fn parse_mixed_plain_and_prefix() { + let q = parse_search_query("polymarket type:tweet").unwrap(); + assert_eq!(q.q.as_deref(), Some("polymarket")); + assert_eq!(q.entity_kind.as_deref(), Some("tweet")); + } + + #[test] + fn parse_unknown_prefix_returns_err() { + let result = parse_search_query("foo:bar"); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "foo"); + } + + #[test] + fn parse_after_before_dates() { + let q = parse_search_query("after:2026-01-01 before:2026-04-01").unwrap(); + assert_eq!(q.after.as_deref(), Some("2026-01-01")); + assert_eq!(q.before.as_deref(), Some("2026-04-01")); + } + + #[test] + fn parse_search_query_tag_prefix() { + let q = parse_search_query("tag:/science/cs").unwrap(); + assert_eq!(q.tag.as_deref(), Some("/science/cs")); + assert_eq!(q.q, None); + } + + // ---- search_entries tests ---- + + fn make_test_db_with_entries() -> rusqlite::Connection { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + database::initialize_schema(&conn).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 2).unwrap(); + + // Entry 1: tweet by source x + let si1 = database::upsert_source_identity( + &conn, "x", "tweet", Some("t-1"), + Some("https://x.com/user/status/1"), + "https://x.com/user/status/1", + ).unwrap(); + database::create_archived_entry( + &conn, + &database::NewEntry { + source_identity_id: si1, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "x".to_string(), + entity_kind: "tweet".to_string(), + title: Some("Polymarket tweet".to_string()), + visibility: "private".to_string(), + representation_kind: "json".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ).unwrap(); + + // Entry 2: web page + let si2 = database::upsert_source_identity( + &conn, "web", "page", Some("page-1"), + Some("https://medium.com/article"), + "https://medium.com/article", + ).unwrap(); + database::create_archived_entry( + &conn, + &database::NewEntry { + source_identity_id: si2, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "web".to_string(), + entity_kind: "page".to_string(), + title: Some("Resume Templates".to_string()), + visibility: "private".to_string(), + representation_kind: "html".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ).unwrap(); + + conn + } + + #[test] + fn search_empty_query_returns_all_root_entries() { + let conn = make_test_db_with_entries(); + let all = list_root_entries(&conn).unwrap(); + let searched = search_entries(&conn, &SearchEntriesQuery::default()).unwrap(); + assert_eq!(all.len(), searched.len()); + } + + #[test] + fn search_q_filters_on_title() { + let conn = make_test_db_with_entries(); + let results = search_entries(&conn, &SearchEntriesQuery { + q: Some("polymarket".to_string()), + ..Default::default() + }).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].entity_kind, "tweet"); + } + + #[test] + fn search_entity_kind_exact_match() { + let conn = make_test_db_with_entries(); + let results = search_entries(&conn, &SearchEntriesQuery { + entity_kind: Some("page".to_string()), + ..Default::default() + }).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].source_kind, "web"); + } + + #[test] + fn search_url_like_filter() { + let conn = make_test_db_with_entries(); + let results = search_entries(&conn, &SearchEntriesQuery { + url: Some("medium.com".to_string()), + ..Default::default() + }).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].title.as_deref(), Some("Resume Templates")); + } + + #[test] + fn search_no_match_returns_empty() { + let conn = make_test_db_with_entries(); + let results = search_entries(&conn, &SearchEntriesQuery { + q: Some("zzznonexistent".to_string()), + ..Default::default() + }).unwrap(); + assert!(results.is_empty()); + } + + #[test] + fn search_multiple_filters_compound() { + let conn = make_test_db_with_entries(); + let results = search_entries(&conn, &SearchEntriesQuery { + source_kind: Some("x".to_string()), + entity_kind: Some("tweet".to_string()), + ..Default::default() + }).unwrap(); + assert_eq!(results.len(), 1); + } + + // ---- tag API tests ---- + + fn make_tag_test_db() -> (rusqlite::Connection, i64, i64) { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + database::initialize_schema(&conn).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 2).unwrap(); + (conn, user_id, run.id) + } + + fn make_entry_in_db( + conn: &rusqlite::Connection, + user_id: i64, + run_id: i64, + parent_entry_id: Option, + root_entry_id: Option, + title: &str, + url: &str, + ) -> database::ArchivedEntry { + let si = database::upsert_source_identity( + conn, "web", "page", None, Some(url), url, + ).unwrap(); + database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id: si, + archive_run_id: run_id, + parent_entry_id, + root_entry_id, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "web".to_string(), + entity_kind: "page".to_string(), + title: Some(title.to_string()), + visibility: "private".to_string(), + representation_kind: "html".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ).unwrap() + } + + #[test] + fn tag_tree_roots_and_children() { + let (conn, _, _) = make_tag_test_db(); + create_tag(&conn, "/science/cs").unwrap(); + create_tag(&conn, "/art").unwrap(); + + let tree = list_tag_tree(&conn).unwrap(); + assert_eq!(tree.len(), 2, "expected two root nodes"); + + let science = tree.iter().find(|n| n.tag.slug == "science").expect("science root missing"); + assert_eq!(science.children.len(), 1, "science should have one child"); + assert_eq!(science.children[0].tag.slug, "cs"); + + let art = tree.iter().find(|n| n.tag.slug == "art").expect("art root missing"); + assert!(art.children.is_empty(), "art should have no children"); + } + + #[test] + fn assign_entry_tag_is_idempotent() { + let (conn, user_id, run_id) = make_tag_test_db(); + let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Test", "https://example.com/t1"); + + assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap(); + assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap(); + + let tags = get_entry_tags(&conn, &entry.entry_uid).unwrap().unwrap(); + assert_eq!(tags.len(), 1, "idempotent assign should yield exactly one tag"); + } + + #[test] + fn remove_entry_tag_clears_assignment() { + let (conn, user_id, run_id) = make_tag_test_db(); + let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Test", "https://example.com/t2"); + + let tag = assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap().unwrap(); + remove_entry_tag(&conn, &entry.entry_uid, &tag.tag_uid).unwrap(); + + let tags = get_entry_tags(&conn, &entry.entry_uid).unwrap().unwrap(); + assert!(tags.is_empty(), "tag should be removed"); + } + + #[test] + fn entries_for_tag_includes_descendants() { + let (conn, user_id, run_id) = make_tag_test_db(); + let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Compilers Paper", "https://example.com/c1"); + + assign_entry_tag(&conn, &entry.entry_uid, "/science/cs/compilers").unwrap(); + + let results = entries_for_tag(&conn, "/science").unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].entry_uid, entry.entry_uid); + } + + #[test] + fn entries_for_tag_includes_child_entries() { + let (conn, user_id, run_id) = make_tag_test_db(); + let parent = make_entry_in_db(&conn, user_id, run_id, None, None, "Playlist", "https://example.com/pl"); + let child = make_entry_in_db( + &conn, user_id, run_id, + Some(parent.id), Some(parent.id), + "Video 1", "https://example.com/pl/v1", + ); + + assign_entry_tag(&conn, &child.entry_uid, "/science").unwrap(); + + let results = entries_for_tag(&conn, "/science").unwrap(); + assert_eq!(results.len(), 1, "only the tagged child should appear"); + assert_eq!(results[0].entry_uid, child.entry_uid); + assert_eq!(results[0].parent_entry_uid.as_deref(), Some(parent.entry_uid.as_str())); + } + + #[test] + fn search_with_tag_filter_works() { + let (conn, user_id, run_id) = make_tag_test_db(); + let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Science Article", "https://example.com/s1"); + + assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap(); + + let results = search_entries(&conn, &SearchEntriesQuery { + tag: Some("/science".to_string()), + ..Default::default() + }).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].entry_uid, entry.entry_uid); + + let empty = search_entries(&conn, &SearchEntriesQuery { + tag: Some("/art".to_string()), + ..Default::default() + }).unwrap(); + assert!(empty.is_empty(), "no entries under /art"); + } + + #[test] + fn search_without_tag_returns_roots_only() { + let (conn, user_id, run_id) = make_tag_test_db(); + let parent = make_entry_in_db(&conn, user_id, run_id, None, None, "Parent", "https://example.com/par"); + let _child = make_entry_in_db( + &conn, user_id, run_id, + Some(parent.id), Some(parent.id), + "Child", "https://example.com/par/c", + ); + + let results = search_entries(&conn, &SearchEntriesQuery::default()).unwrap(); + assert_eq!(results.len(), 1, "plain search should return root entries only"); + assert_eq!(results[0].entry_uid, parent.entry_uid); + } } diff --git a/crates/archivr-core/src/capture.rs b/crates/archivr-core/src/capture.rs new file mode 100644 index 0000000..a1690f7 --- /dev/null +++ b/crates/archivr-core/src/capture.rs @@ -0,0 +1,1261 @@ +use anyhow::{Context, Result}; +use chrono::Local; +use serde_json::json; +use std::{ + collections::HashSet, + fs, + path::{Path, PathBuf}, +}; +use crate::{archive::ArchivePaths, database, downloader, twitter::parse_tweet_id}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum Source { + YouTubeVideo, + YouTubePlaylist, + YouTubeChannel, + X, + Tweet, + TweetThread, + Instagram, + Facebook, + TikTok, + Reddit, + Snapchat, + Local, + Other, +} + +#[derive(Debug, serde::Serialize)] +pub struct CaptureResult { + pub run_uid: String, + pub status: String, +} + +fn expand_shorthand_to_url(path: &str, source: &Source) -> String { + // YouTube shorthands: yt:video/ID, yt:playlist/ID, yt:@handle, yt:channel/ID, etc. + if matches!(source, Source::YouTubeVideo | Source::YouTubePlaylist | Source::YouTubeChannel) { + if let Some(after) = path.strip_prefix("yt:").or_else(|| path.strip_prefix("youtube:")) { + if let Some(id) = after + .strip_prefix("video/") + .or_else(|| after.strip_prefix("short/")) + .or_else(|| after.strip_prefix("shorts/")) + { + return format!("https://www.youtube.com/watch?v={id}"); + } + if let Some(id) = after.strip_prefix("playlist/") { + return format!("https://www.youtube.com/playlist?list={id}"); + } + if let Some(id) = after.strip_prefix("channel/") { + return format!("https://www.youtube.com/channel/{id}"); + } + if let Some(id) = after.strip_prefix("c/") { + return format!("https://www.youtube.com/c/{id}"); + } + if let Some(id) = after.strip_prefix("user/") { + return format!("https://www.youtube.com/user/{id}"); + } + if let Some(handle) = after.strip_prefix("@") { + return format!("https://www.youtube.com/@{handle}"); + } + } + } + + if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) { + if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) { + return format!("https://x.com/i/status/{tweet_id}"); + } + } + + if let Some(path) = path.strip_prefix("instagram:") { + if let Some(id) = path.strip_prefix("reel:") { + return format!("https://www.instagram.com/reel/{id}"); + } + return format!("https://www.instagram.com/{path}"); + } + if let Some(path) = path.strip_prefix("facebook:") { + return format!("https://www.facebook.com/{path}"); + } + if let Some(path) = path.strip_prefix("tiktok:") { + return format!("https://www.tiktok.com/{path}"); + } + if let Some(path) = path.strip_prefix("reddit:") { + return format!("https://www.reddit.com/{path}"); + } + if let Some(path) = path.strip_prefix("snapchat:") { + return format!("https://www.snapchat.com/{path}"); + } + + path.to_string() +} + +// INFO: yt-dlp supports a lot of sites; so, when archiving (for example) a website, the user +// -> should be asked whether they want to archive the whole website or just the video(s) on it. +fn determine_source(path: &str) -> Source { + // INFO: Extractor URLs can be found here: + // -> https://github.com/yt-dlp/yt-dlp/tree/dfc0a84c192a7357dd1768cc345d590253a14fe5/yt_dlp/extractor + // TEST: X posts can have multiple videos. + + // Shorthand schemes: yt: or youtube: + if let Some(after_scheme) = path + .strip_prefix("yt:") + .or_else(|| path.strip_prefix("youtube:")) + { + // video/ID, short/ID, shorts/ID + if after_scheme.starts_with("video/") + || after_scheme.starts_with("short/") + || after_scheme.starts_with("shorts/") + { + return Source::YouTubeVideo; + } + + // playlist/ID + if after_scheme.starts_with("playlist/") { + return Source::YouTubePlaylist; + } + + // channel/ID, c/ID, user/ID, @handle + if after_scheme.starts_with("channel/") + || after_scheme.starts_with("c/") + || after_scheme.starts_with("user/") + || after_scheme.starts_with("@") + { + return Source::YouTubeChannel; + } + } + + // Shorthand schemes: tweet:, x:, or twitter: + if let Some(after_scheme) = path + .strip_prefix("x:") + .or_else(|| path.strip_prefix("twitter:")) + .or_else(|| path.strip_prefix("tweet:")) + { + // For this scope, in comments, N is an alias for a string of type ('twitter' | 'x' | 'tweet'). + + // N:media:id + if after_scheme.starts_with("media:") + && after_scheme + .strip_prefix("media:") + .and_then(parse_tweet_id) + .is_some() + { + return Source::X; + } + + // N:tweet:id or N:x:id + if after_scheme + .strip_prefix("tweet:") + .or_else(|| after_scheme.strip_prefix("x:")) + .and_then(parse_tweet_id) + .is_some() + { + return Source::Tweet; + } + + // N:thread:id + if after_scheme + .strip_prefix("thread:") + .and_then(parse_tweet_id) + .is_some() + { + return Source::TweetThread; + } + + // N:id + if parse_tweet_id(after_scheme).is_some() { + return Source::Tweet; + } + + // N:non-id + return Source::Other; + } + + // Shorthand schemes for other yt-dlp extractors + if path.starts_with("instagram:") { + return Source::Instagram; + } + if path.starts_with("facebook:") { + return Source::Facebook; + } + if path.starts_with("tiktok:") { + return Source::TikTok; + } + if path.starts_with("reddit:") { + return Source::Reddit; + } + if path.starts_with("snapchat:") { + return Source::Snapchat; + } + + if path.starts_with("file://") { + return Source::Local; + } else if path.starts_with("http://") || path.starts_with("https://") { + // Video URLs (watch, youtu.be, shorts) + let video_re = regex::Regex::new(r"^https?://(?:www\.)?(?:youtu\.be/[0-9A-Za-z_-]+|youtube\.com/watch\?v=[0-9A-Za-z_-]+|youtube\.com/shorts/[0-9A-Za-z_-]+)") + .expect("YouTube video URL regex literal must be valid"); + if video_re.is_match(path) { + return Source::YouTubeVideo; + } + + // Playlist URLs + let playlist_re = + regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/playlist\?list=[0-9A-Za-z_-]+") + .expect("YouTube playlist URL regex literal must be valid"); + if playlist_re.is_match(path) { + return Source::YouTubePlaylist; + } + + // Channel or user URLs (channel IDs, /c/, /user/, or @handles) + let channel_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/(?:channel/[0-9A-Za-z_-]+|c/[0-9A-Za-z_-]+|user/[0-9A-Za-z_-]+|@[0-9A-Za-z_-]+)") + .expect("YouTube channel URL regex literal must be valid"); + if channel_re.is_match(path) { + return Source::YouTubeChannel; + } + + if path.starts_with("https://x.com/") { + return Source::X; + } + + if path.starts_with("https://instagram.com/") + || path.starts_with("https://www.instagram.com/") + || path.starts_with("http://instagram.com/") + || path.starts_with("http://www.instagram.com/") + { + return Source::Instagram; + } + + if path.starts_with("https://facebook.com/") + || path.starts_with("https://www.facebook.com/") + || path.starts_with("http://facebook.com/") + || path.starts_with("http://www.facebook.com/") + || path.starts_with("https://fb.watch/") + || path.starts_with("http://fb.watch/") + { + return Source::Facebook; + } + + if path.starts_with("https://tiktok.com/") + || path.starts_with("https://www.tiktok.com/") + || path.starts_with("http://tiktok.com/") + || path.starts_with("http://www.tiktok.com/") + { + return Source::TikTok; + } + + if path.starts_with("https://reddit.com/") + || path.starts_with("https://www.reddit.com/") + || path.starts_with("http://reddit.com/") + || path.starts_with("http://www.reddit.com/") + || path.starts_with("https://redd.it/") + || path.starts_with("http://redd.it/") + { + return Source::Reddit; + } + + if path.starts_with("https://snapchat.com/") + || path.starts_with("https://www.snapchat.com/") + || path.starts_with("http://snapchat.com/") + || path.starts_with("http://www.snapchat.com/") + { + return Source::Snapchat; + } + } + if Path::new(path).exists() { + return Source::Local; + } + Source::Other +} + +fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result { + let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); + + println!("Checking {}", path.display()); + + Ok(path.exists()) +} + +fn move_temp_to_raw(file: &Path, hash: &str, store_path: &Path) -> Result<()> { + let file_extension = file + .extension() + .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())); + let raw_relpath = raw_relative_path_from_hash(hash, &file_extension)?; + let destination = store_path.join(raw_relpath); + + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent)?; + } + + fs::rename(file, destination)?; + + Ok(()) +} + +fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result { + let mut chars = hash.chars(); + let first_letter = chars.next().context("hash must not be empty")?; + let second_letter = chars + .next() + .context("hash must be at least two characters")?; + + Ok(PathBuf::from("raw") + .join(first_letter.to_string()) + .join(second_letter.to_string()) + .join(format!("{hash}{file_extension}"))) +} + +fn path_to_store_string(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +fn extension_without_dot(file_extension: &str) -> Option { + file_extension + .strip_prefix('.') + .filter(|extension| !extension.is_empty()) + .map(|extension| extension.to_string()) +} + +fn blob_record_for_raw_relpath( + store_path: &Path, + raw_relpath: &Path, +) -> Result { + let absolute_path = store_path.join(raw_relpath); + let file_name = raw_relpath + .file_name() + .and_then(|name| name.to_str()) + .context("raw artifact path must have a UTF-8 file name")?; + let (sha256, extension) = match file_name.rsplit_once('.') { + Some((hash, extension)) => (hash.to_string(), Some(extension.to_string())), + None => (file_name.to_string(), None), + }; + + Ok(database::BlobRecord { + sha256, + byte_size: fs::metadata(&absolute_path) + .with_context(|| format!("failed to stat raw artifact {}", absolute_path.display()))? + .len() as i64, + mime_type: None, + extension, + raw_relpath: path_to_store_string(raw_relpath), + }) +} + +fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str) { + match source { + Source::YouTubeVideo => ("youtube", "video", "video"), + Source::YouTubePlaylist => ("youtube", "playlist", "container"), + Source::YouTubeChannel => ("youtube", "channel", "container"), + Source::X => ("x", "post", "video"), + Source::Tweet => ("x", "tweet", "tweet_json"), + Source::TweetThread => ("x", "tweet_thread", "tweet_json"), + Source::Instagram => ("instagram", "post", "video"), + Source::Facebook => ("facebook", "post", "video"), + Source::TikTok => ("tiktok", "video", "video"), + Source::Reddit => ("reddit", "post", "video"), + Source::Snapchat => ("snapchat", "story", "video"), + Source::Local => ("local", "file", "file"), + Source::Other => ("other", "unknown", "unknown"), + } +} + +fn local_file_extension(path: &str) -> String { + Path::new(path.trim_start_matches("file://")) + .extension() + .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) +} + +fn media_file_extension(source: Source, path: &str) -> String { + match source { + Source::YouTubeVideo + | Source::X + | Source::Instagram + | Source::Facebook + | Source::TikTok + | Source::Reddit + | Source::Snapchat => ".mp4".to_string(), + Source::Local => local_file_extension(path), + _ => String::new(), + } +} + +fn tweet_id_from_archive_path(path: &str) -> Option { + path.split(':').next_back().and_then(parse_tweet_id) +} + +fn create_structured_root(store_path: &Path, entry: &database::ArchivedEntry) -> Result<()> { + debug_assert!(entry.entry_uid.starts_with("entry_")); + fs::create_dir_all(store_path.join(&entry.structured_root_relpath))?; + Ok(()) +} + +fn record_media_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + canonical_locator: &str, + source: Source, + hash: &str, + file_extension: &str, + byte_size: i64, +) -> Result { + debug_assert!(run.run_uid.starts_with("run_")); + debug_assert!(item.item_uid.starts_with("item_")); + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let raw_relpath = raw_relative_path_from_hash(hash, file_extension)?; + let blob = database::BlobRecord { + sha256: hash.to_string(), + byte_size, + mime_type: None, + extension: extension_without_dot(file_extension), + raw_relpath: path_to_store_string(&raw_relpath), + }; + let blob_id = database::upsert_blob(conn, &blob)?; + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + None, + Some(canonical_locator), + canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "requested_locator": requested_locator, + "canonical_locator": canonical_locator + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "primary_media".to_string(), + storage_area: "raw".to_string(), + relpath: blob.raw_relpath, + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + )?; + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + +fn record_tweet_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + source: Source, + tweet_id: &str, +) -> Result { + debug_assert!(run.run_uid.starts_with("run_")); + debug_assert!(item.item_uid.starts_with("item_")); + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let canonical_locator = format!("https://x.com/i/status/{tweet_id}"); + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + Some(tweet_id), + Some(&canonical_locator), + &canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "tweet_id": tweet_id, + "requested_locator": requested_locator + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + + let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "raw_tweet_json".to_string(), + storage_area: "raw_tweets".to_string(), + relpath: path_to_store_string(&tweet_json_relpath), + blob_id: None, + logical_path: None, + metadata_json: None, + }, + )?; + + let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; + for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? { + let raw_path = PathBuf::from(&raw_relpath); + let blob = blob_record_for_raw_relpath(store_path, &raw_path)?; + let blob_id = database::upsert_blob(conn, &blob)?; + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: role, + storage_area: "raw".to_string(), + relpath: raw_relpath, + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + )?; + } + + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + +fn tweet_raw_artifacts(tweet_json: &str) -> Result> { + let regex = regex::Regex::new(r#""(avatar_local_path|local_path)": "([^"\n]+)""#)?; + let mut seen = HashSet::new(); + let mut artifacts = Vec::new(); + + for captures in regex.captures_iter(tweet_json) { + let relpath = captures[2].to_string(); + if !relpath.starts_with("raw/") || !seen.insert(relpath.clone()) { + continue; + } + + let role = if &captures[1] == "avatar_local_path" { + "avatar" + } else { + "media" + }; + artifacts.push((role.to_string(), relpath)); + } + + Ok(artifacts) +} + +/// Marks the run and item as failed in the database, returns the error. +/// Call sites: `return Err(fail_run(&conn, &run, &item, "message"));` +fn fail_run( + conn: &rusqlite::Connection, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + message: &str, +) -> anyhow::Error { + let _ = database::fail_archive_run_item(conn, item.id, message); + let _ = database::fail_archive_run(conn, run.id, message); + anyhow::anyhow!("{}", message) +} + +pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result { + let timestamp = Local::now().format("%Y-%m-%dT%H-%M-%S%.3f").to_string(); + let store_path = &archive_paths.store_path; + + let conn = database::open_or_initialize(&archive_paths.archive_path)?; + let user_id = database::ensure_default_user(&conn)?; + + let source = determine_source(locator); + let (source_kind, entity_kind, _) = source_metadata(source); + + let run = database::create_archive_run(&conn, user_id, 1)?; + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + locator, + None, + source_kind, + entity_kind, + )?; + + // Sources: Other (not yet implemented) + if source == Source::Other { + return Err(fail_run( + &conn, + &run, + &item, + "Archiving from this source is not yet implemented.", + )); + } + + // Sources: Tweets or Twitter Threads + if matches!(source, Source::Tweet | Source::TweetThread) { + let tweet_id = match tweet_id_from_archive_path(locator) { + Some(tweet_id) => tweet_id, + None => { + return Err(fail_run( + &conn, + &run, + &item, + "Failed to archive tweet: invalid tweet ID", + )); + } + }; + + match downloader::tweets::archive( + locator, + source == Source::TweetThread, + store_path, + ×tamp, + ) { + Ok(_) => { + record_tweet_entry( + &conn, + store_path, + user_id, + &run, + &item, + locator, + source, + &tweet_id, + )?; + database::finish_archive_run(&conn, run.id)?; + return Ok(CaptureResult { + run_uid: run.run_uid.clone(), + status: "completed".to_string(), + }); + } + Err(e) => { + return Err(fail_run( + &conn, + &run, + &item, + &format!("Failed to archive tweet: {e}"), + )); + } + } + } + + // Sources, for which yt-dlp is needed + let requested_locator = locator.to_string(); + let path = expand_shorthand_to_url(locator, &source); + let hash = match source { + Source::YouTubeVideo + | Source::X + | Source::Instagram + | Source::Facebook + | Source::TikTok + | Source::Reddit + | Source::Snapchat => { + match downloader::ytdlp::download(path.clone(), store_path, ×tamp) { + Ok(h) => h, + Err(e) => { + return Err(fail_run( + &conn, + &run, + &item, + &format!("Failed to download media: {e}"), + )); + } + } + } + Source::Local => { + match downloader::local::save(path.clone(), store_path, ×tamp) { + Ok(h) => h, + Err(e) => { + return Err(fail_run( + &conn, + &run, + &item, + &format!("Failed to archive local file: {e}"), + )); + } + } + } + Source::YouTubePlaylist | Source::YouTubeChannel => { + return Err(fail_run( + &conn, + &run, + &item, + "Playlist and channel container expansion are not yet implemented.", + )); + } + _ => unreachable!(), + }; + + let file_extension = media_file_extension(source, &path); + let temp_file = store_path + .join("temp") + .join(×tamp) + .join(format!("{timestamp}{file_extension}")); + let byte_size = fs::metadata(&temp_file) + .with_context(|| format!("failed to stat staged file {}", temp_file.display()))? + .len() as i64; + + let hash_exists = hash_exists(&hash, &file_extension, store_path)?; + + if hash_exists { + let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); + } else { + move_temp_to_raw( + &store_path + .join("temp") + .join(×tamp) + .join(format!("{timestamp}{file_extension}")), + &hash, + store_path, + )?; + let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); + } + + record_media_entry( + &conn, + store_path, + user_id, + &run, + &item, + &requested_locator, + &path, + source, + &hash, + &file_extension, + byte_size, + )?; + database::finish_archive_run(&conn, run.id)?; + + Ok(CaptureResult { + run_uid: run.run_uid.clone(), + status: "completed".to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{archive, database}; + use chrono::Local; + use std::{env, fs}; + + struct TestCase<'a> { + url: &'a str, + expected: Source, + } + + #[test] + fn test_tweet_sources() { + let cases = [ + TestCase { + url: "tweet:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "x:tweet:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "x:x:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "twitter:x:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "twitter:tweet:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "tweet:media:1234567890", + expected: Source::X, + }, + TestCase { + url: "x:media:1234567890", + expected: Source::X, + }, + TestCase { + url: "x:thread:1234567890", + expected: Source::TweetThread, + }, + TestCase { + url: "twitter:thread:1234567890", + expected: Source::TweetThread, + }, + TestCase { + url: "tweet:thread:1234567890", + expected: Source::TweetThread, + }, + TestCase { + url: "tweet:not-a-number", + expected: Source::Other, + }, + TestCase { + url: "tweet:media:not-a-number", + expected: Source::Other, + }, + TestCase { + url: "x:media:not-a-number", + expected: Source::Other, + }, + ]; + + for case in &cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_resolve_source_path() { + assert_eq!( + expand_shorthand_to_url("tweet:media:1234567890", &Source::X), + "https://x.com/i/status/1234567890" + ); + assert_eq!( + expand_shorthand_to_url("instagram:reel/ABC123", &Source::Instagram), + "https://www.instagram.com/reel/ABC123" + ); + assert_eq!( + expand_shorthand_to_url("facebook:watch?v=123456", &Source::Facebook), + "https://www.facebook.com/watch?v=123456" + ); + assert_eq!( + expand_shorthand_to_url("tiktok:@someone/video/123456789", &Source::TikTok), + "https://www.tiktok.com/@someone/video/123456789" + ); + assert_eq!( + expand_shorthand_to_url("reddit:r/videos/comments/abc123/example", &Source::Reddit), + "https://www.reddit.com/r/videos/comments/abc123/example" + ); + assert_eq!( + expand_shorthand_to_url("snapchat:discover/some-story/1234567890", &Source::Snapchat), + "https://www.snapchat.com/discover/some-story/1234567890" + ); + assert_eq!( + expand_shorthand_to_url("tweet:1234567890", &Source::Tweet), + "tweet:1234567890" + ); + // YouTube shorthands must expand to full URLs before yt-dlp sees them + assert_eq!( + expand_shorthand_to_url("yt:video/MntbN1DdEP0", &Source::YouTubeVideo), + "https://www.youtube.com/watch?v=MntbN1DdEP0" + ); + assert_eq!( + expand_shorthand_to_url("yt:shorts/EtC99eWiwRI", &Source::YouTubeVideo), + "https://www.youtube.com/watch?v=EtC99eWiwRI" + ); + assert_eq!( + expand_shorthand_to_url("youtube:video/UHxw-L2WyyY", &Source::YouTubeVideo), + "https://www.youtube.com/watch?v=UHxw-L2WyyY" + ); + assert_eq!( + expand_shorthand_to_url("yt:playlist/PL9vTTBa7QaQO", &Source::YouTubePlaylist), + "https://www.youtube.com/playlist?list=PL9vTTBa7QaQO" + ); + assert_eq!( + expand_shorthand_to_url("yt:@CoreDumpped", &Source::YouTubeChannel), + "https://www.youtube.com/@CoreDumpped" + ); + assert_eq!( + expand_shorthand_to_url("yt:channel/UCxyz123", &Source::YouTubeChannel), + "https://www.youtube.com/channel/UCxyz123" + ); + // Full YouTube URLs pass through unchanged + assert_eq!( + expand_shorthand_to_url("https://www.youtube.com/watch?v=UHxw-L2WyyY", &Source::YouTubeVideo), + "https://www.youtube.com/watch?v=UHxw-L2WyyY" + ); + } + + #[test] + fn test_youtube_sources() { + // --- YouTube Video URLs --- + let video_cases = [ + TestCase { + url: "https://www.youtube.com/watch?v=UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "https://youtu.be/UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "https://www.youtube.com/shorts/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + ]; + + for case in &video_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + + // --- YouTube Playlist URLs --- + let playlist_cases = [TestCase { + url: "https://www.youtube.com/playlist?list=PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", + expected: Source::YouTubePlaylist, + }]; + + for case in &playlist_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + + // --- YouTube Channel URLs --- + let channel_cases = [ + TestCase { + url: "https://www.youtube.com/channel/CoreDumpped", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://www.youtube.com/@CoreDumpped", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://www.youtube.com/c/YouTubeCreators", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://www.youtube.com/user/pewdiepie", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://youtube.com/@pewdiepie?si=KOcLN_KPYNpe5f_8", + expected: Source::YouTubeChannel, + }, + ]; + + for case in &channel_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + + // --- Shorthand scheme URLs --- + let shorthand_cases = [ + // Videos + TestCase { + url: "yt:video/UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "youtube:video/UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "yt:short/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "yt:shorts/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "youtube:shorts/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + // Playlists + TestCase { + url: "yt:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", + expected: Source::YouTubePlaylist, + }, + TestCase { + url: "youtube:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", + expected: Source::YouTubePlaylist, + }, + // Channels + TestCase { + url: "yt:channel/UCxyz123", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "yt:c/YouTubeCreators", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "yt:user/pewdiepie", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "youtube:@CoreDumpped", + expected: Source::YouTubeChannel, + }, + ]; + + for case in &shorthand_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_x_sources() { + let x_cases = [ + TestCase { + url: "https://x.com/some_post", + expected: Source::X, + }, + TestCase { + url: "x:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "twitter:1234567890", + expected: Source::Tweet, + }, + ]; + + for case in &x_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_other_social_sources() { + let social_cases = [ + TestCase { + url: "https://www.instagram.com/reel/ABC123/", + expected: Source::Instagram, + }, + TestCase { + url: "instagram:reel/ABC123", + expected: Source::Instagram, + }, + TestCase { + url: "https://www.facebook.com/watch/?v=123456", + expected: Source::Facebook, + }, + TestCase { + url: "facebook:watch?v=123456", + expected: Source::Facebook, + }, + TestCase { + url: "https://www.tiktok.com/@someone/video/123456789", + expected: Source::TikTok, + }, + TestCase { + url: "tiktok:@someone/video/123456789", + expected: Source::TikTok, + }, + TestCase { + url: "https://www.reddit.com/r/videos/comments/abc123/example/", + expected: Source::Reddit, + }, + TestCase { + url: "reddit:r/videos/comments/abc123/example", + expected: Source::Reddit, + }, + TestCase { + url: "https://www.snapchat.com/discover/some-story/1234567890", + expected: Source::Snapchat, + }, + TestCase { + url: "snapchat:discover/some-story/1234567890", + expected: Source::Snapchat, + }, + ]; + + for case in &social_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_non_youtube_sources() { + let other_cases = [ + TestCase { + url: "file:///local/path/file.mp4", + expected: Source::Local, + }, + TestCase { + url: "https://example.com/", + expected: Source::Other, + }, + TestCase { + url: "https://example.com/?redirect=instagram.com/reel/ABC123", + expected: Source::Other, + }, + TestCase { + url: "https://notfacebook.com/watch?v=123456", + expected: Source::Other, + }, + ]; + + for case in &other_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_existing_local_path_source() { + let path = env::current_dir().unwrap().join("Cargo.toml"); + assert_eq!( + determine_source(path.to_str().unwrap()), + Source::Local, + "existing filesystem paths should be archived as local files" + ); + } + + #[test] + fn test_initialize_store_directories() { + let store_path = env::temp_dir().join(format!( + "archivr-test-{}", + Local::now().format("%Y%m%d%H%M%S%3f") + )); + + archive::initialize_store_directories(&store_path).unwrap(); + + assert!(store_path.join("raw").is_dir()); + assert!(store_path.join("raw_tweets").is_dir()); + assert!(store_path.join("structured").is_dir()); + assert!(store_path.join("temp").is_dir()); + assert!(!store_path.join("tmp").exists()); + + fs::remove_dir_all(store_path).unwrap(); + } + + #[test] + fn test_record_tweet_entry_links_json_and_raw_artifacts() { + let store_path = env::temp_dir().join(format!( + "archivr-tweet-db-test-{}", + Local::now().format("%Y%m%d%H%M%S%3f") + )); + let _ = fs::remove_dir_all(&store_path); + archive::initialize_store_directories(&store_path).unwrap(); + fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap(); + fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap(); + fs::write( + store_path + .join("raw") + .join("a") + .join("b") + .join("abcdef.jpg"), + b"avatar", + ) + .unwrap(); + fs::write( + store_path + .join("raw") + .join("c") + .join("d") + .join("cdef01.mp4"), + b"media", + ) + .unwrap(); + fs::write( + store_path.join("raw_tweets").join("tweet-123.json"), + r#"{ + "author": { "avatar_local_path": "raw/a/b/abcdef.jpg" }, + "entities": { "media": [{ "local_path": "raw/c/d/cdef01.mp4" }] } +}"#, + ) + .unwrap(); + + let conn = rusqlite::Connection::open_in_memory().unwrap(); + database::initialize_schema(&conn).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 1).unwrap(); + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + "tweet:123", + None, + "x", + "tweet", + ) + .unwrap(); + + let entry = record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + "tweet:123", + Source::Tweet, + "123", + ) + .unwrap(); + database::finish_archive_run(&conn, run.id).unwrap(); + + let artifact_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM entry_artifacts WHERE entry_id = ?1", + [entry.id], + |row| row.get(0), + ) + .unwrap(); + let blob_count: i64 = conn + .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get(0)) + .unwrap(); + let run_status: String = conn + .query_row( + "SELECT status FROM archive_runs WHERE id = ?1", + [run.id], + |row| row.get(0), + ) + .unwrap(); + + assert_eq!(artifact_count, 3); + assert_eq!(blob_count, 2); + assert_eq!(run_status, "completed"); + assert!(store_path.join(&entry.structured_root_relpath).is_dir()); + + let _ = fs::remove_dir_all(store_path); + } +} diff --git a/crates/archivr-core/src/database.rs b/crates/archivr-core/src/database.rs index 5ec3f74..d207c35 100644 --- a/crates/archivr-core/src/database.rs +++ b/crates/archivr-core/src/database.rs @@ -63,6 +63,16 @@ pub struct NewArtifact { pub metadata_json: Option, } +#[derive(Debug, Clone)] +pub struct TagRecord { + pub id: i64, + pub tag_uid: String, + pub parent_tag_id: Option, + pub name: String, + pub slug: String, + pub full_path: String, +} + pub fn database_path(archive_path: &Path) -> PathBuf { archive_path.join(DATABASE_FILE_NAME) } @@ -214,6 +224,7 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> { CREATE INDEX IF NOT EXISTS idx_entry_artifacts_entry_id ON entry_artifacts(entry_id); CREATE INDEX IF NOT EXISTS idx_entry_artifacts_blob_id ON entry_artifacts(blob_id); CREATE INDEX IF NOT EXISTS idx_tags_parent_tag_id ON tags(parent_tag_id); + CREATE INDEX IF NOT EXISTS idx_entry_tag_assignments_tag_id ON entry_tag_assignments(tag_id); "#, )?; Ok(()) @@ -488,6 +499,104 @@ pub fn add_entry_artifact(conn: &Connection, artifact: &NewArtifact) -> Result Result<()> { + conn.execute( + "DELETE FROM entry_tag_assignments WHERE entry_id = ?1 AND tag_id = ?2", + params![entry_id, tag_id], + )?; + Ok(()) +} + +pub fn list_all_tags(conn: &Connection) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, tag_uid, parent_tag_id, name, slug, full_path + FROM tags + ORDER BY full_path", + )?; + let records = stmt + .query_map([], |row| { + Ok(TagRecord { + id: row.get(0)?, + tag_uid: row.get(1)?, + parent_tag_id: row.get(2)?, + name: row.get(3)?, + slug: row.get(4)?, + full_path: row.get(5)?, + }) + })? + .collect::, _>>() + .context("failed to list tags")?; + Ok(records) +} + +pub fn list_tags_for_entry(conn: &Connection, entry_id: i64) -> Result> { + let mut stmt = conn.prepare( + "SELECT t.id, t.tag_uid, t.parent_tag_id, t.name, t.slug, t.full_path + FROM tags t + JOIN entry_tag_assignments eta ON eta.tag_id = t.id + WHERE eta.entry_id = ?1 + ORDER BY t.full_path", + )?; + let records = stmt + .query_map([entry_id], |row| { + Ok(TagRecord { + id: row.get(0)?, + tag_uid: row.get(1)?, + parent_tag_id: row.get(2)?, + name: row.get(3)?, + slug: row.get(4)?, + full_path: row.get(5)?, + }) + })? + .collect::, _>>() + .context("failed to list tags for entry")?; + Ok(records) +} + +pub fn get_tag_by_uid(conn: &Connection, tag_uid: &str) -> Result> { + conn.query_row( + "SELECT id, tag_uid, parent_tag_id, name, slug, full_path + FROM tags WHERE tag_uid = ?1", + [tag_uid], + |row| { + Ok(TagRecord { + id: row.get(0)?, + tag_uid: row.get(1)?, + parent_tag_id: row.get(2)?, + name: row.get(3)?, + slug: row.get(4)?, + full_path: row.get(5)?, + }) + }, + ) + .optional() + .context("failed to get tag by uid") +} + +pub fn get_tag_by_path(conn: &Connection, full_path: &str) -> Result> { + conn.query_row( + "SELECT id, tag_uid, parent_tag_id, name, slug, full_path + FROM tags WHERE full_path = ?1", + [full_path], + |row| { + Ok(TagRecord { + id: row.get(0)?, + tag_uid: row.get(1)?, + parent_tag_id: row.get(2)?, + name: row.get(3)?, + slug: row.get(4)?, + full_path: row.get(5)?, + }) + }, + ) + .optional() + .context("failed to get tag by path") +} + #[cfg(test)] pub fn set_public_settings( conn: &Connection, @@ -535,7 +644,6 @@ pub fn main_archive_entry_count(conn: &Connection) -> Result { Ok(count) } -#[cfg(test)] pub fn create_tag_path(conn: &Connection, full_path: &str) -> Result { let segments = normalized_tag_segments(full_path)?; let mut parent_tag_id = None; @@ -577,7 +685,6 @@ pub fn create_tag_path(conn: &Connection, full_path: &str) -> Result { Ok(current_id) } -#[cfg(test)] pub fn assign_entry_to_tag(conn: &Connection, entry_id: i64, tag_id: i64) -> Result<()> { conn.execute( "INSERT OR IGNORE INTO entry_tag_assignments (entry_id, tag_id) @@ -587,7 +694,6 @@ pub fn assign_entry_to_tag(conn: &Connection, entry_id: i64, tag_id: i64) -> Res Ok(()) } -#[cfg(test)] pub fn entry_count_for_tag_path(conn: &Connection, full_path: &str) -> Result { let count = conn.query_row( "WITH RECURSIVE descendants(id) AS ( @@ -653,7 +759,6 @@ fn validate_visibility(visibility: &str) -> Result<()> { } } -#[cfg(test)] fn normalized_tag_segments(full_path: &str) -> Result> { let segments = full_path .trim() @@ -669,7 +774,6 @@ fn normalized_tag_segments(full_path: &str) -> Result> { Ok(segments) } -#[cfg(test)] fn humanize_slug(slug: &str) -> String { slug.split('-') .map(|part| { diff --git a/crates/archivr-core/src/lib.rs b/crates/archivr-core/src/lib.rs index f1114c1..cd97ffb 100644 --- a/crates/archivr-core/src/lib.rs +++ b/crates/archivr-core/src/lib.rs @@ -1,3 +1,4 @@ +pub mod capture; pub mod archive; pub mod database; pub mod downloader; diff --git a/crates/archivr-server/Cargo.toml b/crates/archivr-server/Cargo.toml index b90f678..4c3548e 100644 --- a/crates/archivr-server/Cargo.toml +++ b/crates/archivr-server/Cargo.toml @@ -10,8 +10,10 @@ axum.workspace = true serde.workspace = true tokio.workspace = true toml.workspace = true +tower.workspace = true tower-http.workspace = true [dev-dependencies] tempfile.workspace = true tower.workspace = true +serde_json.workspace = true diff --git a/crates/archivr-server/src/main.rs b/crates/archivr-server/src/main.rs index 13eb826..47f9532 100644 --- a/crates/archivr-server/src/main.rs +++ b/crates/archivr-server/src/main.rs @@ -1,9 +1,11 @@ mod registry; mod routes; -use anyhow::Result; +use anyhow::{Context, Result}; use std::{net::SocketAddr, path::PathBuf}; +const DEFAULT_BIND: &str = "127.0.0.1:8080"; + #[tokio::main] async fn main() -> Result<()> { let config_path = std::env::args() @@ -11,8 +13,27 @@ async fn main() -> Result<()> { .map(PathBuf::from) .unwrap_or_else(|| PathBuf::from("archivr-server.toml")); let registry = registry::load_registry(&config_path)?; - let app = routes::app(registry); - let addr = SocketAddr::from(([127, 0, 0, 1], 8080)); + let app = routes::app(registry.clone()); + + // Bind address priority: ARCHIVR_BIND env var > TOML bind field > default loopback. + let bind_str = std::env::var("ARCHIVR_BIND") + .ok() + .or_else(|| registry.bind.clone()) + .unwrap_or_else(|| DEFAULT_BIND.to_string()); + + let addr: SocketAddr = bind_str + .parse() + .with_context(|| format!("invalid bind address: {bind_str}"))?; + + // Warn when the server is reachable beyond localhost — it has no authentication. + if !addr.ip().is_loopback() { + eprintln!( + "warn: archivr-server is bound to {addr} — \ + this server has no authentication. \ + Only expose it on a trusted network." + ); + } + let listener = tokio::net::TcpListener::bind(addr).await?; println!("archivr-server listening on http://{addr}"); axum::serve(listener, app).await?; diff --git a/crates/archivr-server/src/registry.rs b/crates/archivr-server/src/registry.rs index c186d01..a9fef6c 100644 --- a/crates/archivr-server/src/registry.rs +++ b/crates/archivr-server/src/registry.rs @@ -14,7 +14,12 @@ pub struct MountedArchive { #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct ServerRegistry { + #[serde(default)] pub archives: Vec, + /// Optional bind address for the server. Defaults to `127.0.0.1:8080`. + /// Set this to `0.0.0.0:8080` only on trusted networks — the server has no authentication. + #[serde(default)] + pub bind: Option, } pub fn load_registry(path: &Path) -> Result { @@ -78,6 +83,7 @@ mod tests { label: "Personal".to_string(), archive_path: archive_path.clone(), }], + bind: None, }; let path = temp.path().join("server.toml"); save_registry(&path, ®istry).unwrap(); @@ -102,10 +108,36 @@ mod tests { archive_path: PathBuf::from("/tmp/b/.archivr"), }, ], + bind: None, }; let err = validate_registry(®istry).unwrap_err().to_string(); assert!(err.contains("duplicate archive id")); } + + #[test] + fn registry_bind_field_round_trips() { + let toml = r#"bind = "127.0.0.1:9090""#; + let registry: ServerRegistry = toml::from_str(toml).unwrap(); + assert_eq!(registry.bind.as_deref(), Some("127.0.0.1:9090")); + assert!(registry.archives.is_empty()); + } + + #[test] + fn registry_bind_field_defaults_to_none_when_absent() { + let toml = r#""#; + let registry: ServerRegistry = toml::from_str(toml).unwrap(); + assert!(registry.bind.is_none()); + } + + #[test] + fn registry_bind_field_does_not_affect_archive_validation() { + let registry = ServerRegistry { + archives: vec![], + bind: Some("0.0.0.0:8080".to_string()), + }; + // validate_registry does not reject non-loopback bind — that's main's concern. + assert!(validate_registry(®istry).is_ok()); + } } diff --git a/crates/archivr-server/src/routes.rs b/crates/archivr-server/src/routes.rs index dff563b..014544a 100644 --- a/crates/archivr-server/src/routes.rs +++ b/crates/archivr-server/src/routes.rs @@ -1,14 +1,39 @@ +// ── Security Boundary ──────────────────────────────────────────────────────── +// All routes are currently trusted-local: no authentication or authorization +// middleware is applied. The server is designed to bind on 127.0.0.1 only. +// +// Route classification (for when middleware is added later): +// +// STATIC — safe to expose publicly: GET / and static /assets/* +// READ — safe to expose read-only: GET /health +// GET /api/archives +// GET /api/archives/:id/entries +// GET /api/archives/:id/entries/search +// GET /api/archives/:id/entries/:uid +// GET /api/archives/:id/entries/:uid/artifacts/:idx +// GET /api/archives/:id/runs +// GET /api/archives/:id/tags +// ADMIN — requires auth if ever public: GET /api/admin/archives +// WRITE — requires auth if ever public: POST /api/archives/:id/captures +// POST /api/archives/:id/tags +// PUT /api/archives/:id/tags/:tag_id +// DELETE /api/archives/:id/tags/:tag_id +// +// Do not add middleware here until the auth model is chosen. See docs/README.md. +// ───────────────────────────────────────────────────────────────────────────── + use std::{path::PathBuf, sync::Arc}; -use archivr_core::{archive, database}; +use archivr_core::{archive, capture, database}; use axum::{ Json, Router, - extract::{Path, State}, + extract::{Path, Query, Request, State}, http::StatusCode, response::{IntoResponse, Response}, - routing::get, + routing::{delete, get, post}, }; use tower_http::services::{ServeDir, ServeFile}; +use tower::ServiceExt; use crate::registry::{MountedArchive, ServerRegistry}; @@ -17,6 +42,12 @@ pub struct AppState { registry: Arc, } +#[derive(Debug, serde::Deserialize, Default)] +pub struct EntrySearchParams { + pub q: Option, + pub tag: Option, +} + pub fn app(registry: ServerRegistry) -> Router { let state = AppState { registry: Arc::new(registry), @@ -27,11 +58,26 @@ pub fn app(registry: ServerRegistry) -> Router { .route("/health", get(|| async { "ok" })) .route("/api/archives", get(list_archives)) .route("/api/archives/:archive_id/entries", get(list_entries)) + .route("/api/archives/:archive_id/entries/search", get(search_entries_handler)) .route( "/api/archives/:archive_id/entries/:entry_uid", get(entry_detail), ) + .route( + "/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index", + get(serve_artifact), + ) .route("/api/archives/:archive_id/runs", get(list_runs)) + .route("/api/archives/:archive_id/captures", post(capture_handler)) + .route("/api/archives/:archive_id/tags", get(list_tags).post(create_tag_handler)) + .route( + "/api/archives/:archive_id/entries/:entry_uid/tags", + get(list_entry_tags).post(assign_entry_tag_handler), + ) + .route( + "/api/archives/:archive_id/entries/:entry_uid/tags/:tag_uid", + delete(remove_entry_tag_handler), + ) .nest_service("/assets", ServeDir::new(&static_dir)) .fallback_service(ServeFile::new(static_dir.join("index.html"))) .with_state(state) @@ -56,6 +102,22 @@ async fn list_entries( Ok(Json(archive::list_root_entries(&conn)?)) } +async fn search_entries_handler( + State(state): State, + Path(archive_id): Path, + Query(params): Query, +) -> Result>, ApiError> { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + let raw = params.q.as_deref().unwrap_or(""); + let mut search_query = archive::parse_search_query(raw) + .map_err(|prefix| ApiError::bad_request(&format!("unknown search prefix: {prefix}")))?; + if let Some(tag) = params.tag { + search_query.tag = Some(tag); + } + Ok(Json(archive::search_entries(&conn, &search_query)?)) +} + async fn entry_detail( State(state): State, Path((archive_id, entry_uid)): Path<(String, String)>, @@ -76,6 +138,125 @@ async fn list_runs( Ok(Json(archive::list_runs(&conn)?)) } +async fn serve_artifact( + State(state): State, + Path((archive_id, entry_uid, artifact_index)): Path<(String, String, usize)>, + req: Request, +) -> Result { + let mounted = mounted_archive(&state, &archive_id)?; + let paths = archive::read_archive_paths(&mounted.archive_path)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + let detail = archive::get_entry_detail(&conn, &entry_uid)? + .ok_or(ApiError::not_found("entry not found"))?; + let artifact = detail + .artifacts + .get(artifact_index) + .ok_or(ApiError::not_found("artifact index out of range"))?; + let file_path = archive::resolve_artifact_path(&paths.store_path, artifact)?; + // ServeFile streams the file, handles Range requests (video seeking), + // sets Content-Type/ETag/Last-Modified. Error type is Infallible. + Ok(ServeFile::new(&file_path) + .oneshot(req) + .await + .unwrap() + .into_response()) +} + +#[derive(Debug, serde::Deserialize)] +struct CreateTagBody { + path: String, +} + +#[derive(Debug, serde::Deserialize)] +struct AssignTagBody { + tag_path: String, +} + +async fn list_tags( + State(state): State, + Path(archive_id): Path, +) -> Result>, ApiError> { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + Ok(Json(archive::list_tag_tree(&conn)?)) +} + +async fn create_tag_handler( + State(state): State, + Path(archive_id): Path, + Json(body): Json, +) -> Result<(StatusCode, Json), ApiError> { + if body.path.trim().is_empty() { + return Err(ApiError::bad_request("tag path must not be empty")); + } + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + let tag = archive::create_tag(&conn, &body.path)?; + Ok((StatusCode::CREATED, Json(tag))) +} + +async fn list_entry_tags( + State(state): State, + Path((archive_id, entry_uid)): Path<(String, String)>, +) -> Result>, ApiError> { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + match archive::get_entry_tags(&conn, &entry_uid)? { + Some(tags) => Ok(Json(tags)), + None => Err(ApiError::not_found("entry not found")), + } +} + +async fn assign_entry_tag_handler( + State(state): State, + Path((archive_id, entry_uid)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json), ApiError> { + if body.tag_path.trim().is_empty() { + return Err(ApiError::bad_request("tag_path must not be empty")); + } + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + match archive::assign_entry_tag(&conn, &entry_uid, &body.tag_path)? { + Some(tag) => Ok((StatusCode::CREATED, Json(tag))), + None => Err(ApiError::not_found("entry not found")), + } +} + +async fn remove_entry_tag_handler( + State(state): State, + Path((archive_id, entry_uid, tag_uid)): Path<(String, String, String)>, +) -> Result { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + if archive::remove_entry_tag(&conn, &entry_uid, &tag_uid)? { + Ok(StatusCode::NO_CONTENT) + } else { + Err(ApiError::not_found("entry or tag not found")) + } +} + +#[derive(Debug, serde::Deserialize)] +struct CaptureBody { + locator: String, +} + +async fn capture_handler( + State(state): State, + Path(archive_id): Path, + Json(body): Json, +) -> Result, ApiError> { + if body.locator.trim().is_empty() { + return Err(ApiError::bad_request("locator must not be empty")); + } + let mounted = mounted_archive(&state, &archive_id)?; + let archive_paths = archive::read_archive_paths(&mounted.archive_path) + .map_err(ApiError::from)?; + let result = capture::perform_capture(&archive_paths, &body.locator) + .map_err(ApiError::from)?; + Ok(Json(result)) +} + fn mounted_archive<'a>( state: &'a AppState, archive_id: &str, @@ -101,6 +282,13 @@ impl ApiError { message: message.to_string(), } } + + fn bad_request(message: &str) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + message: message.to_string(), + } + } } impl From for ApiError @@ -137,6 +325,7 @@ mod tests { label: "Personal".to_string(), archive_path: std::path::PathBuf::from("/tmp/personal/.archivr"), }], + bind: None, }; let response = app(registry) .oneshot( @@ -165,4 +354,627 @@ mod tests { assert_eq!(response.status(), StatusCode::NOT_FOUND); } + + #[tokio::test] + async fn artifact_missing_archive_returns_404() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .uri("/api/archives/nope/entries/entry_abc/artifacts/0") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn artifact_missing_entry_returns_404() { + let dir = tempfile::tempdir().unwrap(); + archivr_core::archive::initialize_archive( + dir.path(), + &dir.path().join("store"), + "test", + false, + ) + .unwrap(); + let archive_path = dir.path().join(".archivr"); + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "test".to_string(), + label: "Test".to_string(), + archive_path, + }], + bind: None, + }; + let response = app(registry) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/entry_doesnotexist/artifacts/0") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn artifact_out_of_range_index_returns_404() { + let dir = tempfile::tempdir().unwrap(); + archivr_core::archive::initialize_archive( + dir.path(), + &dir.path().join("store"), + "test", + false, + ) + .unwrap(); + let archive_path = dir.path().join(".archivr"); + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "test".to_string(), + label: "Test".to_string(), + archive_path, + }], + bind: None, + }; + let response = app(registry) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/entry_doesnotexist/artifacts/99") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn artifact_serves_file_with_ok_status() { + // Initialize archive (creates .archivr dir, store dirs, and db) + let dir = tempfile::tempdir().unwrap(); + let store_path = dir.path().join("store"); + let paths = + archivr_core::archive::initialize_archive(dir.path(), &store_path, "test", false) + .unwrap(); + + // Write artifact file to the store + let artifact_relpath = "raw/a/b/test.html"; + let artifact_dir = store_path.join("raw").join("a").join("b"); + std::fs::create_dir_all(&artifact_dir).unwrap(); + std::fs::write(artifact_dir.join("test.html"), b"hello").unwrap(); + + // Populate the database with user, source identity, run, entry, blob, artifact + let conn = database::open_or_initialize(&paths.archive_path).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let source_identity_id = database::upsert_source_identity( + &conn, + "web", + "page", + Some("test-page"), + Some("https://example.com/page"), + "https://example.com/page", + ) + .unwrap(); + let run = database::create_archive_run(&conn, user_id, 1).unwrap(); + let entry = database::create_archived_entry( + &conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "web".to_string(), + entity_kind: "page".to_string(), + title: Some("Test Page".to_string()), + visibility: "private".to_string(), + representation_kind: "html".to_string(), + source_metadata_json: r#"{"source":"test"}"#.to_string(), + display_metadata_json: None, + }, + ) + .unwrap(); + let blob_id = database::upsert_blob( + &conn, + &database::BlobRecord { + sha256: "abc123testblob".to_string(), + byte_size: 18, + mime_type: Some("text/html".to_string()), + extension: Some("html".to_string()), + raw_relpath: artifact_relpath.to_string(), + }, + ) + .unwrap(); + database::add_entry_artifact( + &conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "primary_media".to_string(), + storage_area: "raw".to_string(), + relpath: artifact_relpath.to_string(), + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + ) + .unwrap(); + drop(conn); // release before the HTTP handler opens the same db file + + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "test".to_string(), + label: "Test".to_string(), + archive_path: paths.archive_path.clone(), + }], + bind: None, + }; + let uri = format!( + "/api/archives/test/entries/{}/artifacts/0", + entry.entry_uid + ); + let response = app(registry) + .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn search_missing_archive_returns_404() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .uri("/api/archives/nope/entries/search?q=anything") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn search_empty_q_returns_ok() { + let dir = tempfile::tempdir().unwrap(); + archivr_core::archive::initialize_archive( + dir.path(), + &dir.path().join("store"), + "test", + false, + ) + .unwrap(); + let archive_path = dir.path().join(".archivr"); + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "test".to_string(), + label: "Test".to_string(), + archive_path, + }], + bind: None, + }; + let response = app(registry) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/search") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn search_unknown_prefix_returns_400() { + let dir = tempfile::tempdir().unwrap(); + archivr_core::archive::initialize_archive( + dir.path(), + &dir.path().join("store"), + "test", + false, + ) + .unwrap(); + let archive_path = dir.path().join(".archivr"); + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "test".to_string(), + label: "Test".to_string(), + archive_path, + }], + bind: None, + }; + let response = app(registry) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/search?q=unknownprefix%3Aval") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + // ---- helpers ---- + + fn make_test_registry(dir: &tempfile::TempDir) -> (ServerRegistry, std::path::PathBuf) { + let paths = archivr_core::archive::initialize_archive( + dir.path(), + &dir.path().join("store"), + "test", + false, + ) + .unwrap(); + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "test".to_string(), + label: "Test".to_string(), + archive_path: paths.archive_path.clone(), + }], + bind: None, + }; + (registry, paths.archive_path) + } + + fn make_test_entry(archive_path: &std::path::Path) -> archivr_core::database::ArchivedEntry { + let conn = database::open_or_initialize(archive_path).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 1).unwrap(); + let si = database::upsert_source_identity( + &conn, "web", "page", None, + Some("https://example.com/test"), + "https://example.com/test", + ) + .unwrap(); + database::create_archived_entry( + &conn, + &database::NewEntry { + source_identity_id: si, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "web".to_string(), + entity_kind: "page".to_string(), + title: Some("Test Entry".to_string()), + visibility: "private".to_string(), + representation_kind: "html".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ) + .unwrap() + } + + async fn body_json(response: axum::response::Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() + } + + fn json_body(payload: &serde_json::Value) -> Body { + Body::from(serde_json::to_vec(payload).unwrap()) + } + + // ---- tag route tests ---- + + #[tokio::test] + async fn test_list_tags_unknown_archive() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .uri("/api/archives/ghost/tags") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn test_create_tag_unknown_archive() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/archives/ghost/tags") + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"path": "/science"}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn test_create_tag_empty_path() { + let dir = tempfile::tempdir().unwrap(); + let (registry, _) = make_test_registry(&dir); + let response = app(registry) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/archives/test/tags") + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"path": ""}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_tag_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let (registry, _) = make_test_registry(&dir); + + let create_response = app(registry.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/archives/test/tags") + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"path": "/science"}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(create_response.status(), StatusCode::CREATED); + + let list_response = app(registry.clone()) + .oneshot( + Request::builder() + .uri("/api/archives/test/tags") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(list_response.status(), StatusCode::OK); + let tree = body_json(list_response).await; + let slugs: Vec<&str> = tree + .as_array() + .unwrap() + .iter() + .map(|n| n["tag"]["slug"].as_str().unwrap()) + .collect(); + assert!(slugs.contains(&"science"), "expected 'science' in tag tree, got {slugs:?}"); + } + + #[tokio::test] + async fn test_entry_tag_assign_and_remove() { + let dir = tempfile::tempdir().unwrap(); + let (registry, archive_path) = make_test_registry(&dir); + let entry = make_test_entry(&archive_path); + let entry_uid = entry.entry_uid.clone(); + let entry_tags_uri = format!("/api/archives/test/entries/{entry_uid}/tags"); + + // Assign tag + let assign_response = app(registry.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri(&entry_tags_uri) + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"tag_path": "/science"}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(assign_response.status(), StatusCode::CREATED); + let assigned_tag = body_json(assign_response).await; + let tag_uid = assigned_tag["tag_uid"].as_str().unwrap().to_string(); + + // List entry tags — should contain the assigned tag + let list_response = app(registry.clone()) + .oneshot( + Request::builder() + .uri(&entry_tags_uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(list_response.status(), StatusCode::OK); + let tags = body_json(list_response).await; + assert_eq!(tags.as_array().unwrap().len(), 1); + + // Remove tag + let delete_uri = format!("{entry_tags_uri}/{tag_uid}"); + let delete_response = app(registry.clone()) + .oneshot( + Request::builder() + .method("DELETE") + .uri(&delete_uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(delete_response.status(), StatusCode::NO_CONTENT); + + // List entry tags again — should be empty + let list2_response = app(registry.clone()) + .oneshot( + Request::builder() + .uri(&entry_tags_uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(list2_response.status(), StatusCode::OK); + let tags2 = body_json(list2_response).await; + assert!(tags2.as_array().unwrap().is_empty(), "tags should be empty after removal"); + } + + #[tokio::test] + async fn test_search_with_tag_param() { + let dir = tempfile::tempdir().unwrap(); + let (registry, archive_path) = make_test_registry(&dir); + let entry = make_test_entry(&archive_path); + let entry_uid = entry.entry_uid.clone(); + + // Assign /science tag to entry + let assign_resp = app(registry.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/archives/test/entries/{entry_uid}/tags")) + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"tag_path": "/science"}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(assign_resp.status(), StatusCode::CREATED, "assign tag should return 201"); + + // Search with ?tag=/science — entry should appear + let response = app(registry.clone()) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/search?tag=%2Fscience") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let results = body_json(response).await; + assert_eq!( + results.as_array().unwrap().len(), + 1, + "expected 1 result for /science tag, got {}", + results.as_array().unwrap().len() + ); + + // Search with ?tag=/art — should return empty + let response2 = app(registry.clone()) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/search?tag=%2Fart") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response2.status(), StatusCode::OK); + let results2 = body_json(response2).await; + assert!( + results2.as_array().unwrap().is_empty(), + "expected 0 results for /art tag" + ); + } + + #[tokio::test] + async fn test_list_entry_tags_unknown_entry() { + let dir = tempfile::tempdir().unwrap(); + let (registry, _) = make_test_registry(&dir); + let response = app(registry) + .oneshot( + Request::builder() + .uri("/api/archives/test/entries/ghost_uid/tags") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn test_assign_entry_tag_unknown_entry() { + let dir = tempfile::tempdir().unwrap(); + let (registry, _) = make_test_registry(&dir); + let response = app(registry) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/archives/test/entries/ghost_uid/tags") + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"tag_path": "/science"}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn test_assign_entry_tag_empty_tag_path() { + let dir = tempfile::tempdir().unwrap(); + let (registry, archive_path) = make_test_registry(&dir); + let entry = make_test_entry(&archive_path); + let entry_uid = entry.entry_uid.clone(); + let response = app(registry) + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/archives/test/entries/{entry_uid}/tags")) + .header("content-type", "application/json") + .body(json_body(&serde_json::json!({"tag_path": ""}))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_remove_entry_tag_unknown_entry() { + let dir = tempfile::tempdir().unwrap(); + let (registry, _) = make_test_registry(&dir); + let response = app(registry) + .oneshot( + Request::builder() + .method("DELETE") + .uri("/api/archives/test/entries/ghost_uid/tags/ghost_tag_uid") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn capture_rejects_empty_locator() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/archives/test/captures") + .header("content-type", "application/json") + .body(Body::from(r#"{"locator":""}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn capture_rejects_unknown_archive() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/archives/nonexistent/captures") + .header("content-type", "application/json") + .body(Body::from(r#"{"locator":"tweet:1234567890"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } } diff --git a/crates/archivr-server/static/app.js b/crates/archivr-server/static/app.js index 68b1616..1934341 100644 --- a/crates/archivr-server/static/app.js +++ b/crates/archivr-server/static/app.js @@ -2,9 +2,11 @@ const state = { archives: [], archiveId: null, entries: [], - filteredEntries: [], selectedEntryUid: null, + selectedEntry: null, + tagFilter: null, }; +let selectSeq = 0; const archiveSwitcher = document.querySelector("#archive-switcher"); const entriesBody = document.querySelector("#entries-body"); @@ -14,6 +16,17 @@ const navButtons = document.querySelectorAll(".nav-link"); const searchInput = document.querySelector("#search"); const resultCount = document.querySelector("#result-count"); const adminArchives = document.querySelector("#admin-archives"); +const tagTree = document.querySelector("#tag-tree"); +const entryTagsEl = document.querySelector("#entry-tags"); +const assignTagForm = document.querySelector("#assign-tag-form"); +const assignTagInput = document.querySelector("#assign-tag-input"); +const assignTagBtn = document.querySelector("#assign-tag-btn"); +const captureButton = document.querySelector('.capture-button'); +const captureDialog = document.querySelector('#capture-dialog'); +const captureLocatorInput = document.querySelector('#capture-locator'); +const captureSubmitBtn = document.querySelector('#capture-submit-btn'); +const captureCancelBtn = document.querySelector('#capture-cancel-btn'); +const captureError = document.querySelector('#capture-error'); function formatBytes(bytes) { if (!bytes) return "0 B"; @@ -71,33 +84,26 @@ function renderArchives() { } } -function applyEntryFilter() { - const query = searchInput.value.trim().toLowerCase(); - if (!query) { - state.filteredEntries = state.entries; - return; - } - state.filteredEntries = state.entries.filter((entry) => { - const haystack = [ - entry.title, - entry.original_url, - entry.entry_uid, - entry.source_kind, - entry.entity_kind, - entry.visibility, - ] - .filter(Boolean) - .join(" ") - .toLowerCase(); - return haystack.includes(query); - }); -} function renderEntries() { entriesBody.innerHTML = ""; - resultCount.textContent = `${state.filteredEntries.length} entries`; + if (state.entries.length === 0 && searchInput.value.trim()) { + resultCount.textContent = "No results."; + } else { + resultCount.textContent = `${state.entries.length} entries`; + } + if (state.tagFilter) { + const badge = document.createElement("button"); + badge.className = "tag-filter-badge"; + badge.textContent = `× ${state.tagFilter}`; + badge.addEventListener("click", () => { + state.tagFilter = null; + if (state.archiveId) loadEntries(searchInput.value); + }); + resultCount.appendChild(badge); + } - for (const entry of state.filteredEntries) { + for (const entry of state.entries) { const row = document.createElement("tr"); row.tabIndex = 0; row.dataset.entryUid = entry.entry_uid; @@ -132,30 +138,139 @@ function renderEntries() { function renderContextDetail(detail) { contextBody.innerHTML = ""; - const title = document.createElement("strong"); - title.textContent = valueText(detail.summary.title) || valueText(detail.summary.entry_uid); - contextBody.append(title); - const items = [ + // Title + const titleEl = document.createElement("strong"); + titleEl.className = "rail-entry-title"; + titleEl.textContent = + valueText(detail.summary.title) || valueText(detail.summary.entry_uid); + contextBody.append(titleEl); + + // Metadata section + const metaSection = document.createElement("div"); + metaSection.className = "rail-section"; + + if (detail.summary.original_url) { + const urlRow = document.createElement("div"); + urlRow.className = "rail-item"; + const urlLabel = document.createElement("span"); + urlLabel.className = "rail-label"; + urlLabel.textContent = "Original URL"; + const urlLink = document.createElement("a"); + urlLink.href = detail.summary.original_url; + urlLink.target = "_blank"; + urlLink.rel = "noopener noreferrer"; + urlLink.className = "rail-url-link"; + urlLink.textContent = detail.summary.original_url; + urlRow.append(urlLabel, document.createTextNode(": "), urlLink); + metaSection.append(urlRow); + } + + const metaFields = [ + ["Added", detail.summary.archived_at], + ["Source", detail.summary.source_kind], ["Type", detail.summary.entity_kind], ["Visibility", detail.summary.visibility], - ["Artifacts", detail.artifacts.length], ["Structured root", detail.structured_root_relpath], ]; - - for (const [label, value] of items) { + for (const [label, value] of metaFields) { const item = document.createElement("div"); item.className = "rail-item"; - item.textContent = `${label}: ${valueText(value)}`; - contextBody.append(item); + const labelEl = document.createElement("span"); + labelEl.className = "rail-label"; + labelEl.textContent = label; + item.append(labelEl, document.createTextNode(`: ${valueText(value)}`)); + metaSection.append(item); + } + contextBody.append(metaSection); + + // Artifacts section + if (detail.artifacts.length > 0) { + const artifactsSection = document.createElement("div"); + artifactsSection.className = "rail-section"; + const artifactsHeading = document.createElement("div"); + artifactsHeading.className = "rail-section-heading"; + artifactsHeading.textContent = `Artifacts (${detail.artifacts.length})`; + artifactsSection.append(artifactsHeading); + const list = document.createElement("ul"); + list.className = "artifact-list"; + detail.artifacts.forEach((artifact, index) => { + const li = document.createElement("li"); + const a = document.createElement("a"); + a.href = `/api/archives/${state.archiveId}/entries/${detail.summary.entry_uid}/artifacts/${index}`; + a.target = "_blank"; + a.rel = "noopener noreferrer"; + a.className = "artifact-link"; + const roleName = artifact.artifact_role.replace(/_/g, " "); + const size = + artifact.byte_size != null ? ` (${formatBytes(artifact.byte_size)})` : ""; + a.textContent = `${roleName}${size}`; + li.append(a); + list.append(li); + }); + artifactsSection.append(list); + contextBody.append(artifactsSection); + } else { + const noArtifacts = document.createElement("div"); + noArtifacts.className = "rail-item muted"; + noArtifacts.textContent = "No artifacts."; + contextBody.append(noArtifacts); + } +} + +function renderEntryTags(tags, entryUid) { + entryTagsEl.innerHTML = ""; + if (!tags.length) { + entryTagsEl.textContent = "No tags."; + return; + } + for (const tag of tags) { + const pill = document.createElement("span"); + pill.className = "tag-pill"; + pill.textContent = tag.name; + pill.title = tag.full_path; + const removeBtn = document.createElement("button"); + removeBtn.className = "remove-tag"; + removeBtn.textContent = "×"; + removeBtn.title = `Remove tag ${tag.full_path}`; + removeBtn.addEventListener("click", async () => { + const resp = await fetch( + `/api/archives/${state.archiveId}/entries/${entryUid}/tags/${tag.tag_uid}`, + { method: "DELETE" } + ); + if (!resp.ok) { + removeBtn.title = `Remove failed (${resp.status})`; + return; + } + const updated = await getJson( + `/api/archives/${state.archiveId}/entries/${entryUid}/tags` + ); + renderEntryTags(updated, entryUid); + loadTagTree(); + }); + pill.appendChild(removeBtn); + entryTagsEl.appendChild(pill); } } async function selectEntry(entry) { + const seq = ++selectSeq; state.selectedEntryUid = entry.entry_uid; + state.selectedEntry = entry; renderEntries(); - const detail = await getJson(`/api/archives/${state.archiveId}/entries/${entry.entry_uid}`); + const detail = await getJson( + `/api/archives/${state.archiveId}/entries/${entry.entry_uid}` + ); + if (seq !== selectSeq) return; renderContextDetail(detail); + entryTagsEl.hidden = false; + assignTagForm.hidden = false; + entryTagsEl.innerHTML = ""; + const tags = await getJson( + `/api/archives/${state.archiveId}/entries/${entry.entry_uid}/tags` + ); + if (seq !== selectSeq) return; + renderEntryTags(tags, entry.entry_uid); } async function loadRuns() { @@ -172,12 +287,68 @@ async function loadRuns() { } } -async function loadEntries() { - state.entries = await getJson(`/api/archives/${state.archiveId}/entries`); - state.selectedEntryUid = null; - applyEntryFilter(); +async function loadEntries(q = "") { + const trimmed = q.trim(); + const params = new URLSearchParams(); + if (trimmed) params.set("q", trimmed); + if (state.tagFilter) params.set("tag", state.tagFilter); + const url = + trimmed || state.tagFilter + ? `/api/archives/${state.archiveId}/entries/search?${params}` + : `/api/archives/${state.archiveId}/entries`; + searchInput.setAttribute("aria-busy", "true"); + try { + state.entries = await getJson(url); + } catch (err) { + resultCount.textContent = "Search failed. Try again."; + state.entries = []; + } finally { + searchInput.removeAttribute("aria-busy"); + } renderEntries(); - contextBody.textContent = "Select an entry."; +} + +async function loadTagTree() { + if (!state.archiveId) return; + const nodes = await getJson(`/api/archives/${state.archiveId}/tags`); + tagTree.innerHTML = ""; + renderTagTree(nodes, tagTree); +} + +function renderTagTree(nodes, container) { + if (!nodes.length) { + container.textContent = "No tags yet."; + return; + } + const ul = document.createElement("ul"); + ul.className = "tag-tree-list"; + for (const node of nodes) { + const li = document.createElement("li"); + const btn = document.createElement("button"); + btn.className = "tag-node-btn"; + if (state.tagFilter === node.tag.full_path) btn.classList.add("is-active"); + btn.textContent = node.tag.name; + btn.title = node.tag.full_path; + btn.addEventListener("click", () => { + if (state.tagFilter === node.tag.full_path) { + state.tagFilter = null; + } else { + state.tagFilter = node.tag.full_path; + } + // Switch to archive view and reload + switchView("archive"); + if (state.archiveId) loadEntries(searchInput.value); + }); + li.appendChild(btn); + if (node.children?.length) { + const childContainer = document.createElement("div"); + childContainer.className = "tag-children"; + renderTagTree(node.children, childContainer); + li.appendChild(childContainer); + } + ul.appendChild(li); + } + container.appendChild(ul); } async function loadArchives() { @@ -187,32 +358,118 @@ async function loadArchives() { if (state.archiveId) { await loadEntries(); await loadRuns(); + loadTagTree(); } else { contextBody.textContent = "No archives mounted."; resultCount.textContent = "0 entries"; } } +function debounce(fn, ms) { + let timer; + return (...args) => { + clearTimeout(timer); + timer = setTimeout(() => fn(...args), ms); + }; +} + archiveSwitcher.addEventListener("change", async () => { + state.tagFilter = null; + state.selectedEntry = null; + state.selectedEntryUid = null; + entryTagsEl.hidden = true; + assignTagForm.hidden = true; state.archiveId = archiveSwitcher.value; await loadEntries(); await loadRuns(); + loadTagTree(); }); +const debouncedSearch = debounce((q) => { + if (state.archiveId) loadEntries(q); +}, 300); + searchInput.addEventListener("input", () => { - applyEntryFilter(); - renderEntries(); + debouncedSearch(searchInput.value); }); +function switchView(name) { + navButtons.forEach(b => b.classList.toggle("is-active", b.dataset.view === name)); + document.querySelectorAll(".view").forEach(v => v.classList.remove("is-active")); + document.querySelector(`#${name}-view`)?.classList.add("is-active"); +} + navButtons.forEach((button) => { button.addEventListener("click", () => { - navButtons.forEach((candidate) => candidate.classList.remove("is-active")); - document.querySelectorAll(".view").forEach((view) => view.classList.remove("is-active")); - button.classList.add("is-active"); - document.querySelector(`#${button.dataset.view}-view`).classList.add("is-active"); + switchView(button.dataset.view); + if (button.dataset.view === "tags") loadTagTree(); }); }); +assignTagBtn.addEventListener("click", async () => { + const path = assignTagInput.value.trim(); + if (!path || !state.selectedEntry) return; + const resp = await fetch( + `/api/archives/${state.archiveId}/entries/${state.selectedEntry.entry_uid}/tags`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ tag_path: path }), + } + ); + if (resp.ok) { + assignTagInput.setCustomValidity(""); + assignTagInput.value = ""; + const tags = await getJson( + `/api/archives/${state.archiveId}/entries/${state.selectedEntry.entry_uid}/tags` + ); + renderEntryTags(tags, state.selectedEntry.entry_uid); + loadTagTree(); + } else { + assignTagInput.setCustomValidity(`Failed to add tag (${resp.status})`); + assignTagInput.reportValidity(); + } +}); + +captureButton.addEventListener('click', () => { + captureLocatorInput.value = ''; + captureError.hidden = true; + captureDialog.showModal(); +}); + +captureCancelBtn.addEventListener('click', () => captureDialog.close()); + +captureSubmitBtn.addEventListener('click', async () => { + const locator = captureLocatorInput.value.trim(); + if (!locator) { + captureError.textContent = 'Enter a locator.'; + captureError.hidden = false; + return; + } + captureSubmitBtn.disabled = true; + captureSubmitBtn.textContent = 'Capturing\u2026'; + captureError.hidden = true; + try { + const res = await fetch(`/api/archives/${state.archiveId}/captures`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ locator }), + }); + if (!res.ok) { + const msg = await res.text(); + throw new Error(msg || `HTTP ${res.status}`); + } + captureDialog.close(); + await Promise.all([loadEntries(searchInput.value), loadRuns()]); + } catch (e) { + captureError.textContent = e.message; + captureError.hidden = false; + } finally { + captureSubmitBtn.disabled = false; + captureSubmitBtn.textContent = 'Capture'; + } +}); + loadArchives().catch((error) => { contextBody.textContent = `Failed to load archives: ${error.message}`; }); diff --git a/crates/archivr-server/static/index.html b/crates/archivr-server/static/index.html index 021ef15..9aa2777 100644 --- a/crates/archivr-server/static/index.html +++ b/crates/archivr-server/static/index.html @@ -14,6 +14,7 @@ + @@ -66,14 +67,37 @@

Mounted Archives

+ +
+
+
+ +
+

Capture

+ + + +
+ + +
+
+ + diff --git a/crates/archivr-server/static/styles.css b/crates/archivr-server/static/styles.css index c0e9216..46f246f 100644 --- a/crates/archivr-server/static/styles.css +++ b/crates/archivr-server/static/styles.css @@ -312,3 +312,277 @@ select { min-width: 860px; } } + +.rail-entry-title { + display: block; + font-size: 15px; + font-weight: 700; + color: var(--ink); + margin-bottom: 12px; + line-height: 1.4; +} + +.rail-section { + margin-bottom: 18px; +} + +.rail-section-heading { + font-size: 11px; + font-weight: 800; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--accent); + margin-bottom: 6px; +} + +.rail-label { + font-weight: 600; + color: var(--ink); +} + +.rail-url-link { + color: var(--accent); + word-break: break-all; + font-size: 13px; +} + +.artifact-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 4px; +} + +.artifact-link { + display: block; + padding: 6px 8px; + background: var(--paper-3); + border: 1px solid var(--line); + color: var(--accent); + text-decoration: none; + font-size: 13px; + border-radius: 3px; +} + +.artifact-link:hover { + background: var(--line); + text-decoration: underline; +} +/* Search loading state */ +.search-input[aria-busy="true"] { + cursor: progress; +} + +/* Tag tree */ +.tag-tree { + padding: 12px; +} + +.tag-tree-list { + list-style: none; + margin: 0; + padding: 0; +} + +.tag-children { + padding-left: 16px; +} + +.tag-node-btn { + border: 0; + background: transparent; + color: var(--link); + cursor: pointer; + padding: 3px 0; + text-align: left; + font-size: 13px; + display: block; + width: 100%; +} + +.tag-node-btn:hover { + text-decoration: underline; +} + +.tag-node-btn.is-active { + font-weight: 600; + color: var(--ink); +} + +/* Tag pills in context rail */ +.entry-tags { + display: flex; + flex-wrap: wrap; + gap: 6px; + padding: 8px 0; +} + +.tag-pill { + display: inline-flex; + align-items: center; + gap: 4px; + background: var(--paper-2); + border: 1px solid var(--line); + border-radius: 3px; + font-size: 12px; + padding: 2px 6px; + color: var(--ink); +} + +.remove-tag { + border: 0; + background: none; + cursor: pointer; + color: var(--muted); + padding: 0; + font-size: 14px; + line-height: 1; +} + +.remove-tag:hover { + color: var(--accent); +} + +/* Assign tag form */ +.assign-tag-form { + display: flex; + gap: 6px; + padding: 6px 0 8px; + border-top: 1px solid var(--line-soft); + margin-top: 4px; +} + +.assign-tag-input { + flex: 1; + min-width: 0; + border: 1px solid var(--line); + background: var(--paper-3); + color: var(--ink); + padding: 4px 7px; + font-size: 12px; +} + +.assign-tag-btn { + border: 1px solid var(--line); + background: var(--paper-2); + color: var(--ink); + cursor: pointer; + padding: 4px 10px; + font-size: 12px; + white-space: nowrap; +} + +.assign-tag-btn:hover { + background: var(--paper); +} + +/* Tag filter badge in search row */ +.tag-filter-badge { + display: inline-flex; + align-items: center; + gap: 4px; + background: var(--accent); + color: #fff; + border: 0; + border-radius: 3px; + font-size: 12px; + padding: 2px 8px; + cursor: pointer; + margin-left: 8px; +} + +.tag-filter-badge:hover { + opacity: 0.85; +} + +/* Capture dialog */ +.capture-dialog { + border: 2px solid var(--ink); + background: var(--paper); + padding: 0; + min-width: 360px; + max-width: 520px; + box-shadow: 4px 4px 0 var(--ink); +} + +.capture-dialog::backdrop { + background: rgba(0, 0, 0, 0.45); +} + +.capture-dialog-inner { + padding: 28px; +} + +.capture-dialog-title { + margin: 0 0 16px; + font-size: 18px; + font-family: Georgia, "Times New Roman", serif; +} + +.capture-label { + display: block; + font-size: 13px; + font-weight: 600; + margin-bottom: 6px; + color: var(--muted); + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.capture-input { + width: 100%; + height: 42px; + border: 2px solid var(--ink); + background: var(--paper-3); + color: var(--ink); + padding: 0 12px; + font-size: 15px; + margin-bottom: 10px; + outline-offset: 3px; +} + +.capture-error { + color: var(--accent); + font-size: 13px; + margin-bottom: 10px; + min-height: 18px; +} + +.capture-actions { + display: flex; + gap: 10px; + justify-content: flex-end; + margin-top: 16px; +} + +.capture-cancel { + border: 1px solid var(--line); + background: none; + color: var(--ink); + padding: 8px 18px; + cursor: pointer; +} + +.capture-cancel:hover { + background: var(--paper-2); +} + +.capture-submit { + border: 0; + background: var(--ink); + color: var(--paper); + padding: 8px 20px; + cursor: pointer; + font-weight: 600; +} + +.capture-submit:hover { + opacity: 0.85; +} + +.capture-submit:disabled { + opacity: 0.45; + cursor: default; +} diff --git a/docs/README.md b/docs/README.md index c6d1eba..7917645 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,79 @@ This project aims to provide a reliable solution for archiving important data fr - Direct platform URLs - Platform shorthand inputs such as `tweet:...`, `yt:...`, or `instagram:...` +## Running Archivr + +Archivr currently ships as two binaries: + +- `archivr` + - The CLI for creating and writing to one archive. + - Use this for `init` and `archive`. +- `archivr-server` + - The web server for reading one or more existing archives through the browser UI. + - Use this after archives already exist. + +With Nix, run the CLI with: + +```sh +nix run .#archivr -- init ./my-archive --name "My Archive" +nix run .#archivr -- archive file:///absolute/path/to/file.pdf +``` + +Run the web server with: + +```sh +nix run .#archivr-server -- ./archivr-server.toml +``` + +The server expects a TOML registry file. If no path is passed, it reads `./archivr-server.toml`. + +Example: + +```toml +[[archives]] +id = "personal" +label = "Personal" +archive_path = "/absolute/path/to/my-archive/.archivr" +``` + +Then open: + +```text +http://127.0.0.1:8080 +``` + +When installed through Nix, `archivr-server` is wrapped so it can find the static web UI assets automatically. The wrapper sets `ARCHIVR_STATIC_DIR` to the installed static asset directory. Running from source with `cargo run -p archivr-server` falls back to `crates/archivr-server/static`. + +### Security and Deployment + +`archivr-server` is a **local-only tool by default**. It binds to `127.0.0.1:8080` and has no authentication or access control. Do not expose it to a public network or a shared LAN without understanding the risks. + +**Changing the bind address** + +You can set the bind address in your TOML config: + +```toml +# Optional. Default: 127.0.0.1:8080 +# Only change this if you know what you are doing — the server has no authentication. +bind = "127.0.0.1:9090" +``` + +Or override it with the `ARCHIVR_BIND` environment variable: + +```sh +ARCHIVR_BIND=127.0.0.1:9090 nix run .#archivr-server -- ./archivr-server.toml +``` + +If the server is started with a non-loopback address (e.g. `0.0.0.0`), it prints a warning to stderr: + +```text +warn: archivr-server is bound to 0.0.0.0:8080 — this server has no authentication. Only expose it on a trusted network. +``` + +**When will auth be added?** + +Auth and session handling will be designed when remote or public hosting becomes a real requirement. Until then, keep the server on loopback. See `crates/archivr-server/src/routes.rs` for the route classification that will guide where middleware is applied. + ### Supported Platforms - Local files: `file:///absolute/path/to/file.ext` diff --git a/flake.nix b/flake.nix index ca122fb..368dd8c 100644 --- a/flake.nix +++ b/flake.nix @@ -129,18 +129,31 @@ pname = "archivr-server-wrapped"; inherit version; nativeBuildInputs = [ pkgs.makeWrapper ]; + buildInputs = [ tweetPython ]; phases = [ "installPhase" ]; installPhase = '' mkdir -p $out/bin $out/libexec/archivr-server $out/share/archivr-server/static cp ${archivr_server_unwrapped}/bin/archivr-server $out/libexec/archivr-server/archivr-server + cp ${./vendor/twitter/scrape_user_tweet_contents.py} $out/libexec/archivr-server/scrape_user_tweet_contents.py + chmod +x $out/libexec/archivr-server/scrape_user_tweet_contents.py cp -r ${./crates/archivr-server/static}/* $out/share/archivr-server/static/ makeWrapper $out/libexec/archivr-server/archivr-server $out/bin/archivr-server \ - --set ARCHIVR_STATIC_DIR $out/share/archivr-server/static + --set ARCHIVR_STATIC_DIR $out/share/archivr-server/static \ + --set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \ + --set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr-server/scrape_user_tweet_contents.py ''; }; + archivr-all = pkgs.symlinkJoin { + name = "archivr-all"; + paths = [ + archivr + archivr_server + ]; + }; in { - default = archivr; + default = archivr-all; + archivr-all = archivr-all; archivr = archivr; archivr-cli = archivr; archivr-cli-unwrapped = archivr_cli_unwrapped;