diff --git a/.gitignore b/.gitignore index 817505d..8f160bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,8 @@ !.gitignore -!docs/ -!docs/LICENSE -!docs/README* +!docs +!docs/** !crates !crates/** diff --git a/ARCHIVR-MENTAL-MODEL.md b/ARCHIVR-MENTAL-MODEL.md index f383ff1..01f4387 100644 --- a/ARCHIVR-MENTAL-MODEL.md +++ b/ARCHIVR-MENTAL-MODEL.md @@ -2,13 +2,6 @@ This document explains the current project shape after the workspace refactor. -## Key Documents - -| Document | Role | -|---|---| -| `ARCHIVR-MENTAL-MODEL.md` | **This file.** Current architecture, data flows, and where to edit. | -| `docs/README.md` | User-facing docs: how to run the tool, supported inputs, environment variables. | - ## The Big Model Archivr is now a Rust workspace with three crates: @@ -64,40 +57,6 @@ label = "Personal" archive_path = "/path/to/archive/.archivr" ``` -## How To Run It - -There are two user-facing binaries: - -| Binary | Purpose | -|---|---| -| `archivr` | CLI for initializing archives and capturing material into one archive | -| `archivr-server` | Web server for browsing one or more existing archives | - -The CLI writes archive data: - -```sh -nix run .#archivr -- init ./my-archive --name "My Archive" -nix run .#archivr -- archive file:///absolute/path/to/file.pdf -``` - -The server reads archive data: - -```sh -nix run .#archivr-server -- ./archivr-server.toml -``` - -If no config path is passed, the server reads `./archivr-server.toml`. -The config is a server registry, not archive data: - -```toml -[[archives]] -id = "personal" -label = "Personal" -archive_path = "/absolute/path/to/my-archive/.archivr" -``` - -The packaged Nix server wrapper sets `ARCHIVR_STATIC_DIR` so the server can find the installed web UI assets. Source-tree runs do not need that variable because they fall back to `crates/archivr-server/static`. - ## Write Data Flow When archiving something through the CLI: @@ -171,6 +130,6 @@ The web server reads archive data and serves the UI. It does not yet implement c Search is currently simple client-side filtering. -**Auth and session model:** The server binds to `127.0.0.1` by default and has no authentication middleware. This is intentional — Archivr is a local tool. The bind address is configurable via the TOML `bind` field or `ARCHIVR_BIND` env var; a non-loopback address triggers a startup warning. Route families are classified (READ / ADMIN / WRITE / STATIC) in `crates/archivr-server/src/routes.rs` as the decision record for when middleware is eventually added. See the "Security and Deployment" section in `docs/README.md`. +Auth is not a production model yet. Admin is a mounted-archives view, not a management system. diff --git a/Cargo.lock b/Cargo.lock index e220753..5a6b629 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,6 @@ dependencies = [ "archivr-core", "axum", "serde", - "serde_json", "tempfile", "tokio", "toml", diff --git a/crates/archivr-cli/src/main.rs b/crates/archivr-cli/src/main.rs index 4cf56a4..d47c9c9 100644 --- a/crates/archivr-cli/src/main.rs +++ b/crates/archivr-cli/src/main.rs @@ -1,9 +1,12 @@ use anyhow::{Context, Result}; -use archivr_core::archive; +use archivr_core::{archive, database, downloader, twitter::parse_tweet_id}; +use chrono::Local; use clap::{Parser, Subcommand}; +use serde_json::json; use std::{ - env, - path::Path, + collections::HashSet, + env, fs, + path::{Path, PathBuf}, process, }; @@ -50,6 +53,539 @@ enum Command { }, } +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum Source { + YouTubeVideo, + YouTubePlaylist, + YouTubeChannel, + X, + Tweet, + TweetThread, + Instagram, + Facebook, + TikTok, + Reddit, + Snapchat, + Local, + Other, +} + +fn expand_shorthand_to_url(path: &str, source: &Source) -> String { + if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) { + if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) { + return format!("https://x.com/i/status/{tweet_id}"); + } + } + + if let Some(path) = path.strip_prefix("instagram:") { + if let Some(id) = path.strip_prefix("reel:") { + return format!("https://www.instagram.com/reel/{id}"); + } + return format!("https://www.instagram.com/{path}"); + } + if let Some(path) = path.strip_prefix("facebook:") { + return format!("https://www.facebook.com/{path}"); + } + if let Some(path) = path.strip_prefix("tiktok:") { + return format!("https://www.tiktok.com/{path}"); + } + if let Some(path) = path.strip_prefix("reddit:") { + return format!("https://www.reddit.com/{path}"); + } + if let Some(path) = path.strip_prefix("snapchat:") { + return format!("https://www.snapchat.com/{path}"); + } + + path.to_string() +} + +// INFO: yt-dlp supports a lot of sites; so, when archiving (for example) a website, the user +// -> should be asked whether they want to archive the whole website or just the video(s) on it. +fn determine_source(path: &str) -> Source { + // INFO: Extractor URLs can be found here: + // -> https://github.com/yt-dlp/yt-dlp/tree/dfc0a84c192a7357dd1768cc345d590253a14fe5/yt_dlp/extractor + // TEST: X posts can have multiple videos. + + // Shorthand schemes: yt: or youtube: + if let Some(after_scheme) = path + .strip_prefix("yt:") + .or_else(|| path.strip_prefix("youtube:")) + { + // video/ID, short/ID, shorts/ID + if after_scheme.starts_with("video/") + || after_scheme.starts_with("short/") + || after_scheme.starts_with("shorts/") + { + return Source::YouTubeVideo; + } + + // playlist/ID + if after_scheme.starts_with("playlist/") { + return Source::YouTubePlaylist; + } + + // channel/ID, c/ID, user/ID, @handle + if after_scheme.starts_with("channel/") + || after_scheme.starts_with("c/") + || after_scheme.starts_with("user/") + || after_scheme.starts_with("@") + { + return Source::YouTubeChannel; + } + } + + // Shorthand schemes: tweet:, x:, or twitter: + if let Some(after_scheme) = path + .strip_prefix("x:") + .or_else(|| path.strip_prefix("twitter:")) + .or_else(|| path.strip_prefix("tweet:")) + { + // For this scope, in comments, N is an alias for a string of type ('twitter' | 'x' | 'tweet'). + + // N:media:id + if after_scheme.starts_with("media:") + && after_scheme + .strip_prefix("media:") + .and_then(parse_tweet_id) + .is_some() + { + return Source::X; + } + + // N:tweet:id or N:x:id + if after_scheme + .strip_prefix("tweet:") + .or_else(|| after_scheme.strip_prefix("x:")) + .and_then(parse_tweet_id) + .is_some() + { + return Source::Tweet; + } + + // N:thread:id + if after_scheme + .strip_prefix("thread:") + .and_then(parse_tweet_id) + .is_some() + { + return Source::TweetThread; + } + + // N:id + if parse_tweet_id(after_scheme).is_some() { + return Source::Tweet; + } + + // N:non-id + return Source::Other; + } + + // Shorthand schemes for other yt-dlp extractors + if path.starts_with("instagram:") { + return Source::Instagram; + } + if path.starts_with("facebook:") { + return Source::Facebook; + } + if path.starts_with("tiktok:") { + return Source::TikTok; + } + if path.starts_with("reddit:") { + return Source::Reddit; + } + if path.starts_with("snapchat:") { + return Source::Snapchat; + } + + if path.starts_with("file://") { + return Source::Local; + } else if path.starts_with("http://") || path.starts_with("https://") { + // Video URLs (watch, youtu.be, shorts) + let video_re = regex::Regex::new(r"^https?://(?:www\.)?(?:youtu\.be/[0-9A-Za-z_-]+|youtube\.com/watch\?v=[0-9A-Za-z_-]+|youtube\.com/shorts/[0-9A-Za-z_-]+)") + .expect("YouTube video URL regex literal must be valid"); + if video_re.is_match(path) { + return Source::YouTubeVideo; + } + + // Playlist URLs + let playlist_re = + regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/playlist\?list=[0-9A-Za-z_-]+") + .expect("YouTube playlist URL regex literal must be valid"); + if playlist_re.is_match(path) { + return Source::YouTubePlaylist; + } + + // Channel or user URLs (channel IDs, /c/, /user/, or @handles) + let channel_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/(?:channel/[0-9A-Za-z_-]+|c/[0-9A-Za-z_-]+|user/[0-9A-Za-z_-]+|@[0-9A-Za-z_-]+)") + .expect("YouTube channel URL regex literal must be valid"); + if channel_re.is_match(path) { + return Source::YouTubeChannel; + } + + if path.starts_with("https://x.com/") { + return Source::X; + } + + if path.starts_with("https://instagram.com/") + || path.starts_with("https://www.instagram.com/") + || path.starts_with("http://instagram.com/") + || path.starts_with("http://www.instagram.com/") + { + return Source::Instagram; + } + + if path.starts_with("https://facebook.com/") + || path.starts_with("https://www.facebook.com/") + || path.starts_with("http://facebook.com/") + || path.starts_with("http://www.facebook.com/") + || path.starts_with("https://fb.watch/") + || path.starts_with("http://fb.watch/") + { + return Source::Facebook; + } + + if path.starts_with("https://tiktok.com/") + || path.starts_with("https://www.tiktok.com/") + || path.starts_with("http://tiktok.com/") + || path.starts_with("http://www.tiktok.com/") + { + return Source::TikTok; + } + + if path.starts_with("https://reddit.com/") + || path.starts_with("https://www.reddit.com/") + || path.starts_with("http://reddit.com/") + || path.starts_with("http://www.reddit.com/") + || path.starts_with("https://redd.it/") + || path.starts_with("http://redd.it/") + { + return Source::Reddit; + } + + if path.starts_with("https://snapchat.com/") + || path.starts_with("https://www.snapchat.com/") + || path.starts_with("http://snapchat.com/") + || path.starts_with("http://www.snapchat.com/") + { + return Source::Snapchat; + } + } + if Path::new(path).exists() { + return Source::Local; + } + Source::Other +} + +fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result { + let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); + + println!("Checking {}", path.display()); + + Ok(path.exists()) +} + +fn move_temp_to_raw(file: &Path, hash: &str, store_path: &Path) -> Result<()> { + let file_extension = file + .extension() + .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())); + let raw_relpath = raw_relative_path_from_hash(hash, &file_extension)?; + let destination = store_path.join(raw_relpath); + + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent)?; + } + + fs::rename(file, destination)?; + + Ok(()) +} + +fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result { + let mut chars = hash.chars(); + let first_letter = chars.next().context("hash must not be empty")?; + let second_letter = chars + .next() + .context("hash must be at least two characters")?; + + Ok(PathBuf::from("raw") + .join(first_letter.to_string()) + .join(second_letter.to_string()) + .join(format!("{hash}{file_extension}"))) +} + +fn path_to_store_string(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +fn extension_without_dot(file_extension: &str) -> Option { + file_extension + .strip_prefix('.') + .filter(|extension| !extension.is_empty()) + .map(|extension| extension.to_string()) +} + +fn blob_record_for_raw_relpath( + store_path: &Path, + raw_relpath: &Path, +) -> Result { + let absolute_path = store_path.join(raw_relpath); + let file_name = raw_relpath + .file_name() + .and_then(|name| name.to_str()) + .context("raw artifact path must have a UTF-8 file name")?; + let (sha256, extension) = match file_name.rsplit_once('.') { + Some((hash, extension)) => (hash.to_string(), Some(extension.to_string())), + None => (file_name.to_string(), None), + }; + + Ok(database::BlobRecord { + sha256, + byte_size: fs::metadata(&absolute_path) + .with_context(|| format!("failed to stat raw artifact {}", absolute_path.display()))? + .len() as i64, + mime_type: None, + extension, + raw_relpath: path_to_store_string(raw_relpath), + }) +} + +fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str) { + match source { + Source::YouTubeVideo => ("youtube", "video", "video"), + Source::YouTubePlaylist => ("youtube", "playlist", "container"), + Source::YouTubeChannel => ("youtube", "channel", "container"), + Source::X => ("x", "post", "video"), + Source::Tweet => ("x", "tweet", "tweet_json"), + Source::TweetThread => ("x", "tweet_thread", "tweet_json"), + Source::Instagram => ("instagram", "post", "video"), + Source::Facebook => ("facebook", "post", "video"), + Source::TikTok => ("tiktok", "video", "video"), + Source::Reddit => ("reddit", "post", "video"), + Source::Snapchat => ("snapchat", "story", "video"), + Source::Local => ("local", "file", "file"), + Source::Other => ("other", "unknown", "unknown"), + } +} + +fn local_file_extension(path: &str) -> String { + Path::new(path.trim_start_matches("file://")) + .extension() + .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) +} + +fn media_file_extension(source: Source, path: &str) -> String { + match source { + Source::YouTubeVideo + | Source::X + | Source::Instagram + | Source::Facebook + | Source::TikTok + | Source::Reddit + | Source::Snapchat => ".mp4".to_string(), + Source::Local => local_file_extension(path), + _ => String::new(), + } +} + +fn tweet_id_from_archive_path(path: &str) -> Option { + path.split(':').next_back().and_then(parse_tweet_id) +} + +fn create_structured_root(store_path: &Path, entry: &database::ArchivedEntry) -> Result<()> { + debug_assert!(entry.entry_uid.starts_with("entry_")); + fs::create_dir_all(store_path.join(&entry.structured_root_relpath))?; + Ok(()) +} + +fn record_media_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + canonical_locator: &str, + source: Source, + hash: &str, + file_extension: &str, + byte_size: i64, +) -> Result { + debug_assert!(run.run_uid.starts_with("run_")); + debug_assert!(item.item_uid.starts_with("item_")); + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let raw_relpath = raw_relative_path_from_hash(hash, file_extension)?; + let blob = database::BlobRecord { + sha256: hash.to_string(), + byte_size, + mime_type: None, + extension: extension_without_dot(file_extension), + raw_relpath: path_to_store_string(&raw_relpath), + }; + let blob_id = database::upsert_blob(conn, &blob)?; + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + None, + Some(canonical_locator), + canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "requested_locator": requested_locator, + "canonical_locator": canonical_locator + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "primary_media".to_string(), + storage_area: "raw".to_string(), + relpath: blob.raw_relpath, + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + )?; + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + +fn record_tweet_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + source: Source, + tweet_id: &str, +) -> Result { + debug_assert!(run.run_uid.starts_with("run_")); + debug_assert!(item.item_uid.starts_with("item_")); + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let canonical_locator = format!("https://x.com/i/status/{tweet_id}"); + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + Some(tweet_id), + Some(&canonical_locator), + &canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "tweet_id": tweet_id, + "requested_locator": requested_locator + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + + let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "raw_tweet_json".to_string(), + storage_area: "raw_tweets".to_string(), + relpath: path_to_store_string(&tweet_json_relpath), + blob_id: None, + logical_path: None, + metadata_json: None, + }, + )?; + + let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; + for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? { + let raw_path = PathBuf::from(&raw_relpath); + let blob = blob_record_for_raw_relpath(store_path, &raw_path)?; + let blob_id = database::upsert_blob(conn, &blob)?; + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: role, + storage_area: "raw".to_string(), + relpath: raw_relpath, + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + )?; + } + + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + +fn tweet_raw_artifacts(tweet_json: &str) -> Result> { + let regex = regex::Regex::new(r#""(avatar_local_path|local_path)": "([^"\n]+)""#)?; + let mut seen = HashSet::new(); + let mut artifacts = Vec::new(); + + for captures in regex.captures_iter(tweet_json) { + let relpath = captures[2].to_string(); + if !relpath.starts_with("raw/") || !seen.insert(relpath.clone()) { + continue; + } + + let role = if &captures[1] == "avatar_local_path" { + "avatar" + } else { + "media" + }; + artifacts.push((role.to_string(), relpath)); + } + + Ok(artifacts) +} + +fn fail_archive_and_exit( + conn: &rusqlite::Connection, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + message: &str, +) -> ! { + let _ = database::fail_archive_run_item(conn, item.id, message); + let _ = database::fail_archive_run(conn, run.id, message); + eprintln!("{message}"); + process::exit(1); +} + fn main() -> Result<()> { let args = Args::parse(); @@ -62,9 +598,214 @@ fn main() -> Result<()> { process::exit(1); } }; - let archive_paths = archive::read_archive_paths(&archive_path)?; - let result = archivr_core::capture::perform_capture(&archive_paths, path)?; - println!("Archived: run {}", result.run_uid); + + // let download_id = uuid::Uuid::new_v4(); + let timestamp = Local::now().format("%Y-%m-%dT%H-%M-%S%.3f").to_string(); + + let store_path_string_file = archive_path.join("store_path"); + let store_path = match fs::read_to_string(store_path_string_file) { + Ok(p) => PathBuf::from(p.trim()), + Err(e) => { + eprintln!("Failed to read store path: {e}"); + process::exit(1); + } + }; + + let source = determine_source(path); + let (source_kind, entity_kind, _) = source_metadata(source); + let conn = database::open_or_initialize(&archive_path)?; + let user_id = database::ensure_default_user(&conn)?; + let run = database::create_archive_run(&conn, user_id, 1)?; + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + path, + None, + source_kind, + entity_kind, + )?; + + // Sources: Tweets or Twitter Threads + match source { + Source::Other => { + fail_archive_and_exit( + &conn, + &run, + &item, + "Archiving from this source is not yet implemented.", + ); + } + Source::Tweet | Source::TweetThread => { + let tweet_id = match tweet_id_from_archive_path(path) { + Some(tweet_id) => tweet_id, + None => fail_archive_and_exit( + &conn, + &run, + &item, + "Failed to archive tweet: invalid tweet ID", + ), + }; + + match downloader::tweets::archive( + path, + source == Source::TweetThread, + &store_path, + ×tamp, + ) { + Ok(true) => { + record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + path, + source, + &tweet_id, + )?; + database::finish_archive_run(&conn, run.id)?; + println!( + "Tweet archived successfully to {}", + store_path.join("raw_tweets").display() + ); + return Ok(()); + } + Ok(false) => { + record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + path, + source, + &tweet_id, + )?; + database::finish_archive_run(&conn, run.id)?; + println!( + "Tweet already archived in {}", + store_path.join("raw_tweets").display() + ); + return Ok(()); + } + Err(e) => { + fail_archive_and_exit( + &conn, + &run, + &item, + &format!("Failed to archive tweet: {e}"), + ); + } + } + } + _ => {} + } + + // Sources, for which yt-dlp is needed + let requested_path = path.to_string(); + let path = expand_shorthand_to_url(path, &source); + let hash = match source { + Source::YouTubeVideo + | Source::X + | Source::Instagram + | Source::Facebook + | Source::TikTok + | Source::Reddit + | Source::Snapchat => { + match downloader::ytdlp::download(path.clone(), &store_path, ×tamp) { + Ok(h) => h, + Err(e) => { + fail_archive_and_exit( + &conn, + &run, + &item, + &format!("Failed to download media: {e}"), + ); + } + } + } + Source::Local => { + match downloader::local::save(path.clone(), &store_path, ×tamp) { + Ok(h) => h, + Err(e) => { + fail_archive_and_exit( + &conn, + &run, + &item, + &format!("Failed to archive local file: {e}"), + ); + } + } + } + Source::YouTubePlaylist | Source::YouTubeChannel => { + fail_archive_and_exit( + &conn, + &run, + &item, + "Playlist and channel container expansion are not yet implemented.", + ); + } + _ => unreachable!(), + }; + + let file_extension = media_file_extension(source, &path); + let temp_file = store_path + .join("temp") + .join(×tamp) + .join(format!("{timestamp}{file_extension}")); + let byte_size = fs::metadata(&temp_file) + .with_context(|| format!("failed to stat staged file {}", temp_file.display()))? + .len() as i64; + + let hash_exists = hash_exists(&hash, &file_extension, &store_path)?; + + // TODO: check for repeated archives? + // There could be one of the following: + // - We are literally archiving the same path over again. + // - We are archiving a different path, which had this file. E.g.: we archived a + // website before which had this YouTube video, and while recursively archiving + // everything, we also archived the YouTube video although it wasn't our main + // target. This means that we should archive again; whereas with the first case... + // Not sure. Need to think about this. + // ---- + // Thinking about it a day later... + // If we are specifically archiving a YouTube video, it could also be two of the + // above. So yeah, just create a new DB entry and symlink the Raw to the Structured + // Dir or whatever. it's midnight and my brain ain't wording/braining. + if hash_exists { + println!("File already archived."); + let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); + } else { + move_temp_to_raw( + &store_path + .join("temp") + .join(×tamp) + .join(format!("{timestamp}{file_extension}")), + &hash, + &store_path, + )?; + let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); + + println!("File archived successfully."); + } + + record_media_entry( + &conn, + &store_path, + user_id, + &run, + &item, + &requested_path, + &path, + source, + &hash, + &file_extension, + byte_size, + )?; + database::finish_archive_run(&conn, run.id)?; + Ok(()) } @@ -100,3 +841,481 @@ fn main() -> Result<()> { } } +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + struct TestCase<'a> { + url: &'a str, + expected: Source, + } + + #[test] + fn test_tweet_sources() { + let cases = [ + TestCase { + url: "tweet:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "x:tweet:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "x:x:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "twitter:x:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "twitter:tweet:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "tweet:media:1234567890", + expected: Source::X, + }, + TestCase { + url: "x:media:1234567890", + expected: Source::X, + }, + TestCase { + url: "x:thread:1234567890", + expected: Source::TweetThread, + }, + TestCase { + url: "twitter:thread:1234567890", + expected: Source::TweetThread, + }, + TestCase { + url: "tweet:thread:1234567890", + expected: Source::TweetThread, + }, + TestCase { + url: "tweet:not-a-number", + expected: Source::Other, + }, + TestCase { + url: "tweet:media:not-a-number", + expected: Source::Other, + }, + TestCase { + url: "x:media:not-a-number", + expected: Source::Other, + }, + ]; + + for case in &cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_resolve_source_path() { + assert_eq!( + expand_shorthand_to_url("tweet:media:1234567890", &Source::X), + "https://x.com/i/status/1234567890" + ); + assert_eq!( + expand_shorthand_to_url("instagram:reel/ABC123", &Source::Instagram), + "https://www.instagram.com/reel/ABC123" + ); + assert_eq!( + expand_shorthand_to_url("facebook:watch?v=123456", &Source::Facebook), + "https://www.facebook.com/watch?v=123456" + ); + assert_eq!( + expand_shorthand_to_url("tiktok:@someone/video/123456789", &Source::TikTok), + "https://www.tiktok.com/@someone/video/123456789" + ); + assert_eq!( + expand_shorthand_to_url("reddit:r/videos/comments/abc123/example", &Source::Reddit), + "https://www.reddit.com/r/videos/comments/abc123/example" + ); + assert_eq!( + expand_shorthand_to_url("snapchat:discover/some-story/1234567890", &Source::Snapchat), + "https://www.snapchat.com/discover/some-story/1234567890" + ); + assert_eq!( + expand_shorthand_to_url("tweet:1234567890", &Source::Tweet), + "tweet:1234567890" + ); + } + + #[test] + fn test_youtube_sources() { + // --- YouTube Video URLs --- + let video_cases = [ + TestCase { + url: "https://www.youtube.com/watch?v=UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "https://youtu.be/UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "https://www.youtube.com/shorts/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + ]; + + for case in &video_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + + // --- YouTube Playlist URLs --- + let playlist_cases = [TestCase { + url: "https://www.youtube.com/playlist?list=PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", + expected: Source::YouTubePlaylist, + }]; + + for case in &playlist_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + + // --- YouTube Channel URLs --- + let channel_cases = [ + TestCase { + url: "https://www.youtube.com/channel/CoreDumpped", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://www.youtube.com/@CoreDumpped", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://www.youtube.com/c/YouTubeCreators", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://www.youtube.com/user/pewdiepie", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "https://youtube.com/@pewdiepie?si=KOcLN_KPYNpe5f_8", + expected: Source::YouTubeChannel, + }, + ]; + + for case in &channel_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + + // --- Shorthand scheme URLs --- + let shorthand_cases = [ + // Videos + TestCase { + url: "yt:video/UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "youtube:video/UHxw-L2WyyY", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "yt:short/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "yt:shorts/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + TestCase { + url: "youtube:shorts/EtC99eWiwRI", + expected: Source::YouTubeVideo, + }, + // Playlists + TestCase { + url: "yt:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", + expected: Source::YouTubePlaylist, + }, + TestCase { + url: "youtube:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", + expected: Source::YouTubePlaylist, + }, + // Channels + TestCase { + url: "yt:channel/UCxyz123", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "yt:c/YouTubeCreators", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "yt:user/pewdiepie", + expected: Source::YouTubeChannel, + }, + TestCase { + url: "youtube:@CoreDumpped", + expected: Source::YouTubeChannel, + }, + ]; + + for case in &shorthand_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_x_sources() { + let x_cases = [ + TestCase { + url: "https://x.com/some_post", + expected: Source::X, + }, + TestCase { + url: "x:1234567890", + expected: Source::Tweet, + }, + TestCase { + url: "twitter:1234567890", + expected: Source::Tweet, + }, + ]; + + for case in &x_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_other_social_sources() { + let social_cases = [ + TestCase { + url: "https://www.instagram.com/reel/ABC123/", + expected: Source::Instagram, + }, + TestCase { + url: "instagram:reel/ABC123", + expected: Source::Instagram, + }, + TestCase { + url: "https://www.facebook.com/watch/?v=123456", + expected: Source::Facebook, + }, + TestCase { + url: "facebook:watch?v=123456", + expected: Source::Facebook, + }, + TestCase { + url: "https://www.tiktok.com/@someone/video/123456789", + expected: Source::TikTok, + }, + TestCase { + url: "tiktok:@someone/video/123456789", + expected: Source::TikTok, + }, + TestCase { + url: "https://www.reddit.com/r/videos/comments/abc123/example/", + expected: Source::Reddit, + }, + TestCase { + url: "reddit:r/videos/comments/abc123/example", + expected: Source::Reddit, + }, + TestCase { + url: "https://www.snapchat.com/discover/some-story/1234567890", + expected: Source::Snapchat, + }, + TestCase { + url: "snapchat:discover/some-story/1234567890", + expected: Source::Snapchat, + }, + ]; + + for case in &social_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_non_youtube_sources() { + let other_cases = [ + TestCase { + url: "file:///local/path/file.mp4", + expected: Source::Local, + }, + TestCase { + url: "https://example.com/", + expected: Source::Other, + }, + TestCase { + url: "https://example.com/?redirect=instagram.com/reel/ABC123", + expected: Source::Other, + }, + TestCase { + url: "https://notfacebook.com/watch?v=123456", + expected: Source::Other, + }, + ]; + + for case in &other_cases { + assert_eq!( + determine_source(case.url), + case.expected, + "Failed for URL: {}", + case.url + ); + } + } + + #[test] + fn test_existing_local_path_source() { + let path = env::current_dir().unwrap().join("Cargo.toml"); + assert_eq!( + determine_source(path.to_str().unwrap()), + Source::Local, + "existing filesystem paths should be archived as local files" + ); + } + + #[test] + fn test_initialize_store_directories() { + let store_path = env::temp_dir().join(format!( + "archivr-test-{}", + Local::now().format("%Y%m%d%H%M%S%3f") + )); + + archive::initialize_store_directories(&store_path).unwrap(); + + assert!(store_path.join("raw").is_dir()); + assert!(store_path.join("raw_tweets").is_dir()); + assert!(store_path.join("structured").is_dir()); + assert!(store_path.join("temp").is_dir()); + assert!(!store_path.join("tmp").exists()); + + fs::remove_dir_all(store_path).unwrap(); + } + + #[test] + fn test_record_tweet_entry_links_json_and_raw_artifacts() { + let store_path = env::temp_dir().join(format!( + "archivr-tweet-db-test-{}", + Local::now().format("%Y%m%d%H%M%S%3f") + )); + let _ = fs::remove_dir_all(&store_path); + archive::initialize_store_directories(&store_path).unwrap(); + fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap(); + fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap(); + fs::write( + store_path + .join("raw") + .join("a") + .join("b") + .join("abcdef.jpg"), + b"avatar", + ) + .unwrap(); + fs::write( + store_path + .join("raw") + .join("c") + .join("d") + .join("cdef01.mp4"), + b"media", + ) + .unwrap(); + fs::write( + store_path.join("raw_tweets").join("tweet-123.json"), + r#"{ + "author": { "avatar_local_path": "raw/a/b/abcdef.jpg" }, + "entities": { "media": [{ "local_path": "raw/c/d/cdef01.mp4" }] } +}"#, + ) + .unwrap(); + + let conn = rusqlite::Connection::open_in_memory().unwrap(); + database::initialize_schema(&conn).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 1).unwrap(); + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + "tweet:123", + None, + "x", + "tweet", + ) + .unwrap(); + + let entry = record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + "tweet:123", + Source::Tweet, + "123", + ) + .unwrap(); + database::finish_archive_run(&conn, run.id).unwrap(); + + let artifact_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM entry_artifacts WHERE entry_id = ?1", + [entry.id], + |row| row.get(0), + ) + .unwrap(); + let blob_count: i64 = conn + .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get(0)) + .unwrap(); + let run_status: String = conn + .query_row( + "SELECT status FROM archive_runs WHERE id = ?1", + [run.id], + |row| row.get(0), + ) + .unwrap(); + + assert_eq!(artifact_count, 3); + assert_eq!(blob_count, 2); + assert_eq!(run_status, "completed"); + assert!(store_path.join(&entry.structured_root_relpath).is_dir()); + + let _ = fs::remove_dir_all(store_path); + } +} diff --git a/crates/archivr-core/src/archive.rs b/crates/archivr-core/src/archive.rs index be14c24..0d507c0 100644 --- a/crates/archivr-core/src/archive.rs +++ b/crates/archivr-core/src/archive.rs @@ -25,7 +25,6 @@ pub struct EntrySummary { pub original_url: Option, pub artifact_count: i64, pub total_artifact_bytes: i64, - pub parent_entry_uid: Option, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] @@ -58,20 +57,6 @@ pub struct RunSummary { pub error_summary: Option, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] -pub struct Tag { - pub tag_uid: String, - pub name: String, - pub slug: String, - pub full_path: String, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct TagNode { - pub tag: Tag, - pub children: Vec, -} - pub fn find_archive_path_from(start: &Path) -> Result> { let mut dir = start.to_path_buf(); loop { @@ -182,8 +167,7 @@ pub fn list_root_entries(conn: &rusqlite::Connection) -> Result Result>>()?; @@ -295,384 +278,6 @@ pub fn list_runs(conn: &rusqlite::Connection) -> Result> { Ok(runs) } -/// Resolves an artifact to its absolute on-disk path under `store_path`. -/// -/// `artifact.relpath` is a store-relative path (e.g. `raw/a/b/abc.pdf`). -/// The returned path is canonicalized. Returns an error if the resolved path -/// escapes `store_path` (path traversal protection) or if the file does not exist. -pub fn resolve_artifact_path( - store_path: &Path, - artifact: &EntryArtifactSummary, -) -> Result { - let joined = store_path.join(&artifact.relpath); - let canonical_store = store_path - .canonicalize() - .with_context(|| format!("failed to canonicalize store path: {}", store_path.display()))?; - let canonical_artifact = joined - .canonicalize() - .with_context(|| format!("artifact path does not exist: {}", joined.display()))?; - if !canonical_artifact.starts_with(&canonical_store) { - bail!( - "artifact path escapes store: {}", - canonical_artifact.display() - ); - } - Ok(canonical_artifact) -} - -#[derive(Debug, Clone, Default)] -pub struct SearchEntriesQuery { - /// Free-text term: LIKE-matched against title, canonical_url, entry_uid, source_kind, entity_kind, visibility - pub q: Option, - /// Exact match on e.source_kind - pub source_kind: Option, - /// Exact match on e.entity_kind - pub entity_kind: Option, - /// LIKE-matched against si.canonical_url - pub url: Option, - /// LIKE-matched against e.title - pub title: Option, - /// e.archived_at >= after (inclusive, ISO 8601) - pub after: Option, - /// e.archived_at < before (exclusive, ISO 8601) - pub before: Option, - /// Tag full_path filter; includes all entries (root + child) matching the tag subtree - pub tag: Option, -} - -/// Parses a raw search string into a [`SearchEntriesQuery`]. -/// -/// Recognized prefixes: `source:`, `type:`, `url:`, `title:`, `after:`, `before:`, `tag:`. -/// Tokens with an unrecognized `prefix:` return `Err(prefix)`. -/// Remaining non-prefix tokens are joined as the free-text `q`. -/// Quoted values (`title:"resume templates"`) are supported for single-word values -/// after the colon; leading/trailing double quotes are stripped. -pub fn parse_search_query(raw: &str) -> Result { - let mut query = SearchEntriesQuery::default(); - let mut free_text_tokens: Vec<&str> = Vec::new(); - - for token in raw.split_whitespace() { - if let Some(colon_pos) = token.find(':') { - let prefix = &token[..colon_pos]; - let value_raw = &token[colon_pos + 1..]; - // Strip surrounding double quotes if present - let value = value_raw.trim_matches('"').to_string(); - - match prefix { - "source" => query.source_kind = Some(value), - "type" => query.entity_kind = Some(value), - "url" => query.url = Some(value), - "title" => query.title = Some(value), - "after" => query.after = Some(value), - "before" => query.before = Some(value), - "tag" => query.tag = Some(value), - other => return Err(other.to_string()), - } - } else { - free_text_tokens.push(token); - } - } - - let q = free_text_tokens.join(" "); - query.q = if q.is_empty() { None } else { Some(q) }; - - Ok(query) -} - -const ENTRY_SELECT_COLS: &str = - "SELECT e.entry_uid, e.archived_at, e.source_kind, e.entity_kind, e.title, \ - e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, \ - COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \ - parent.entry_uid AS parent_entry_uid"; - -const ENTRY_FROM_JOINS: &str = - "FROM archived_entries e \ - JOIN source_identities si ON si.id = e.source_identity_id \ - LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id \ - LEFT JOIN blobs b ON b.id = ea.blob_id \ - LEFT JOIN archived_entries parent ON parent.id = e.parent_entry_id"; - -/// Searches archived entries matching all non-`None` fields in `query`. -/// -/// Without a tag filter, returns root entries only (same scope as [`list_root_entries`]). -/// With a tag filter, returns ALL entries (root and child) assigned to that tag subtree. -pub fn search_entries( - conn: &rusqlite::Connection, - query: &SearchEntriesQuery, -) -> Result> { - let mut params: Vec = Vec::new(); - let mut sql; - - if let Some(tag_path) = &query.tag { - params.push(tag_path.clone()); - sql = format!( - "WITH RECURSIVE descendants(id) AS (\ - SELECT id FROM tags WHERE full_path = ?1 \ - UNION ALL \ - SELECT child.id FROM tags child \ - JOIN descendants d ON child.parent_tag_id = d.id\ - ) {} {} \ - JOIN entry_tag_assignments eta ON eta.entry_id = e.id \ - JOIN descendants d ON eta.tag_id = d.id \ - WHERE 1=1", - ENTRY_SELECT_COLS, - ENTRY_FROM_JOINS, - ); - } else { - sql = format!( - "{} {} WHERE e.parent_entry_id IS NULL", - ENTRY_SELECT_COLS, - ENTRY_FROM_JOINS, - ); - } - - if let Some(q) = &query.q { - let term = format!("%{}%", q.to_lowercase()); - let n = params.len() + 1; - sql.push_str(&format!( - " AND (LOWER(e.title) LIKE ?{n} OR LOWER(si.canonical_url) LIKE ?{n} \ - OR LOWER(e.entry_uid) LIKE ?{n} OR LOWER(e.source_kind) LIKE ?{n} \ - OR LOWER(e.entity_kind) LIKE ?{n} OR LOWER(e.visibility) LIKE ?{n})" - )); - params.push(term); - } - if let Some(sk) = &query.source_kind { - let n = params.len() + 1; - sql.push_str(&format!(" AND e.source_kind = ?{n}")); - params.push(sk.clone()); - } - if let Some(ek) = &query.entity_kind { - let n = params.len() + 1; - sql.push_str(&format!(" AND e.entity_kind = ?{n}")); - params.push(ek.clone()); - } - if let Some(u) = &query.url { - let n = params.len() + 1; - sql.push_str(&format!(" AND LOWER(si.canonical_url) LIKE ?{n}")); - params.push(format!("%{}%", u.to_lowercase())); - } - if let Some(t) = &query.title { - let n = params.len() + 1; - sql.push_str(&format!(" AND LOWER(e.title) LIKE ?{n}")); - params.push(format!("%{}%", t.to_lowercase())); - } - if let Some(a) = &query.after { - let n = params.len() + 1; - sql.push_str(&format!(" AND e.archived_at >= ?{n}")); - params.push(a.clone()); - } - if let Some(b) = &query.before { - let n = params.len() + 1; - sql.push_str(&format!(" AND e.archived_at < ?{n}")); - params.push(b.clone()); - } - - sql.push_str(" GROUP BY e.id ORDER BY e.archived_at DESC, e.id DESC"); - - let mut stmt = conn.prepare(&sql)?; - let entries = stmt - .query_map(rusqlite::params_from_iter(params.iter()), |row| { - Ok(EntrySummary { - entry_uid: row.get(0)?, - archived_at: row.get(1)?, - source_kind: row.get(2)?, - entity_kind: row.get(3)?, - title: row.get(4)?, - visibility: row.get(5)?, - original_url: row.get(6)?, - artifact_count: row.get(7)?, - total_artifact_bytes: row.get(8)?, - parent_entry_uid: row.get(9)?, - }) - })? - .collect::>>()?; - Ok(entries) -} - -fn tag_by_id(conn: &rusqlite::Connection, id: i64) -> Result { - conn.query_row( - "SELECT tag_uid, name, slug, full_path FROM tags WHERE id = ?1", - [id], - |row| { - Ok(Tag { - tag_uid: row.get(0)?, - name: row.get(1)?, - slug: row.get(2)?, - full_path: row.get(3)?, - }) - }, - ) - .context("tag not found by id") -} - -/// Creates all tag path segments (idempotent) and returns the leaf `Tag`. -pub fn create_tag(conn: &rusqlite::Connection, full_path: &str) -> Result { - let id = database::create_tag_path(conn, full_path)?; - tag_by_id(conn, id) -} - -/// Returns the full tag tree with root nodes at the top level and children nested. -pub fn list_tag_tree(conn: &rusqlite::Connection) -> Result> { - use std::collections::HashMap; - - let records = database::list_all_tags(conn)?; - - let mut by_parent: HashMap, Vec> = HashMap::new(); - for record in records { - by_parent.entry(record.parent_tag_id).or_default().push(record); - } - - fn build_nodes( - parent_id: Option, - by_parent: &HashMap, Vec>, - ) -> Vec { - let Some(children) = by_parent.get(&parent_id) else { - return Vec::new(); - }; - children - .iter() - .map(|r| TagNode { - tag: Tag { - tag_uid: r.tag_uid.clone(), - name: r.name.clone(), - slug: r.slug.clone(), - full_path: r.full_path.clone(), - }, - children: build_nodes(Some(r.id), by_parent), - }) - .collect() - } - - Ok(build_nodes(None, &by_parent)) -} - -/// Returns the tags assigned to an entry. -/// -/// Returns `Ok(None)` if the entry_uid does not exist (caller maps to 404). -/// Returns `Ok(Some([]))` if the entry exists but has no tags. -pub fn get_entry_tags( - conn: &rusqlite::Connection, - entry_uid: &str, -) -> Result>> { - let Some(entry_id) = conn - .query_row( - "SELECT id FROM archived_entries WHERE entry_uid = ?1", - [entry_uid], - |row| row.get::<_, i64>(0), - ) - .optional()? - else { - return Ok(None); - }; - let records = database::list_tags_for_entry(conn, entry_id)?; - Ok(Some( - records - .into_iter() - .map(|r| Tag { - tag_uid: r.tag_uid, - name: r.name, - slug: r.slug, - full_path: r.full_path, - }) - .collect(), - )) -} - -/// Assigns a tag (by full path, creating it if needed) to an entry. -/// -/// Returns `Ok(None)` if the entry_uid does not exist. -/// Returns `Ok(Some(tag))` on success. -pub fn assign_entry_tag( - conn: &rusqlite::Connection, - entry_uid: &str, - tag_full_path: &str, -) -> Result> { - let Some(entry_id) = conn - .query_row( - "SELECT id FROM archived_entries WHERE entry_uid = ?1", - [entry_uid], - |row| row.get::<_, i64>(0), - ) - .optional()? - else { - return Ok(None); - }; - let tag_id = database::create_tag_path(conn, tag_full_path)?; - database::assign_entry_to_tag(conn, entry_id, tag_id)?; - Ok(Some(tag_by_id(conn, tag_id)?)) -} - -/// Removes a tag assignment from an entry. -/// -/// Returns `Ok(false)` if either the entry_uid or tag_uid is not found. -/// Returns `Ok(true)` on success (even if no row was deleted, i.e. assignment didn't exist). -pub fn remove_entry_tag( - conn: &rusqlite::Connection, - entry_uid: &str, - tag_uid: &str, -) -> Result { - let Some(entry_id) = conn - .query_row( - "SELECT id FROM archived_entries WHERE entry_uid = ?1", - [entry_uid], - |row| row.get::<_, i64>(0), - ) - .optional()? - else { - return Ok(false); - }; - let Some(tag_record) = database::get_tag_by_uid(conn, tag_uid)? else { - return Ok(false); - }; - database::remove_entry_tag_assignment(conn, entry_id, tag_record.id)?; - Ok(true) -} - -/// Returns all entries (root and child) assigned to any tag in the subtree rooted at `tag_full_path`. -pub fn entries_for_tag( - conn: &rusqlite::Connection, - tag_full_path: &str, -) -> Result> { - let mut stmt = conn.prepare( - "WITH RECURSIVE descendants(id) AS ( - SELECT id FROM tags WHERE full_path = ?1 - UNION ALL - SELECT child.id FROM tags child - JOIN descendants d ON child.parent_tag_id = d.id - ) - SELECT e.entry_uid, e.archived_at, e.source_kind, e.entity_kind, e.title, - e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, - COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, - parent.entry_uid AS parent_entry_uid - FROM archived_entries e - JOIN source_identities si ON si.id = e.source_identity_id - LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id - LEFT JOIN blobs b ON b.id = ea.blob_id - LEFT JOIN archived_entries parent ON parent.id = e.parent_entry_id - JOIN entry_tag_assignments eta ON eta.entry_id = e.id - JOIN descendants d ON eta.tag_id = d.id - GROUP BY e.id - ORDER BY e.archived_at DESC, e.id DESC", - )?; - let entries = stmt - .query_map([tag_full_path], |row| { - Ok(EntrySummary { - entry_uid: row.get(0)?, - archived_at: row.get(1)?, - source_kind: row.get(2)?, - entity_kind: row.get(3)?, - title: row.get(4)?, - visibility: row.get(5)?, - original_url: row.get(6)?, - artifact_count: row.get(7)?, - total_artifact_bytes: row.get(8)?, - parent_entry_uid: row.get(9)?, - }) - })? - .collect::>>()?; - Ok(entries) -} - #[cfg(test)] mod tests { use super::*; @@ -830,367 +435,4 @@ mod tests { assert_eq!(runs.len(), 1); assert_eq!(runs[0].status, "completed"); } - - #[test] - fn resolve_artifact_path_returns_absolute_path_within_store() { - let root = unique_path("archivr-resolve-artifact"); - let store_path = root.join("store"); - fs::create_dir_all(store_path.join("raw/a/b")).unwrap(); - let artifact_file = store_path.join("raw/a/b/abc.pdf"); - fs::write(&artifact_file, b"data").unwrap(); - - let artifact = EntryArtifactSummary { - artifact_role: "primary".to_string(), - storage_area: "raw".to_string(), - relpath: "raw/a/b/abc.pdf".to_string(), - byte_size: Some(4), - }; - let resolved = resolve_artifact_path(&store_path, &artifact).unwrap(); - assert_eq!(resolved, artifact_file.canonicalize().unwrap()); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn resolve_artifact_path_rejects_traversal() { - let root = unique_path("archivr-resolve-traversal"); - let store_path = root.join("store"); - fs::create_dir_all(&store_path).unwrap(); - let artifact = EntryArtifactSummary { - artifact_role: "primary".to_string(), - storage_area: "raw".to_string(), - relpath: "../escaped.txt".to_string(), - byte_size: None, - }; - assert!(resolve_artifact_path(&store_path, &artifact).is_err()); - let _ = fs::remove_dir_all(&root); - } - - // ---- parse_search_query tests ---- - - #[test] - fn parse_empty_query_returns_default() { - let q = parse_search_query("").unwrap(); - assert!(q.q.is_none()); - assert!(q.source_kind.is_none()); - assert!(q.entity_kind.is_none()); - } - - #[test] - fn parse_plain_text_sets_q() { - let q = parse_search_query("polymarket").unwrap(); - assert_eq!(q.q.as_deref(), Some("polymarket")); - } - - #[test] - fn parse_prefix_source_sets_source_kind() { - let q = parse_search_query("source:x").unwrap(); - assert_eq!(q.source_kind.as_deref(), Some("x")); - assert!(q.q.is_none()); - } - - #[test] - fn parse_prefix_type_sets_entity_kind() { - let q = parse_search_query("type:tweet").unwrap(); - assert_eq!(q.entity_kind.as_deref(), Some("tweet")); - } - - #[test] - fn parse_mixed_plain_and_prefix() { - let q = parse_search_query("polymarket type:tweet").unwrap(); - assert_eq!(q.q.as_deref(), Some("polymarket")); - assert_eq!(q.entity_kind.as_deref(), Some("tweet")); - } - - #[test] - fn parse_unknown_prefix_returns_err() { - let result = parse_search_query("foo:bar"); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "foo"); - } - - #[test] - fn parse_after_before_dates() { - let q = parse_search_query("after:2026-01-01 before:2026-04-01").unwrap(); - assert_eq!(q.after.as_deref(), Some("2026-01-01")); - assert_eq!(q.before.as_deref(), Some("2026-04-01")); - } - - #[test] - fn parse_search_query_tag_prefix() { - let q = parse_search_query("tag:/science/cs").unwrap(); - assert_eq!(q.tag.as_deref(), Some("/science/cs")); - assert_eq!(q.q, None); - } - - // ---- search_entries tests ---- - - fn make_test_db_with_entries() -> rusqlite::Connection { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - database::initialize_schema(&conn).unwrap(); - let user_id = database::ensure_default_user(&conn).unwrap(); - let run = database::create_archive_run(&conn, user_id, 2).unwrap(); - - // Entry 1: tweet by source x - let si1 = database::upsert_source_identity( - &conn, "x", "tweet", Some("t-1"), - Some("https://x.com/user/status/1"), - "https://x.com/user/status/1", - ).unwrap(); - database::create_archived_entry( - &conn, - &database::NewEntry { - source_identity_id: si1, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: "x".to_string(), - entity_kind: "tweet".to_string(), - title: Some("Polymarket tweet".to_string()), - visibility: "private".to_string(), - representation_kind: "json".to_string(), - source_metadata_json: "{}".to_string(), - display_metadata_json: None, - }, - ).unwrap(); - - // Entry 2: web page - let si2 = database::upsert_source_identity( - &conn, "web", "page", Some("page-1"), - Some("https://medium.com/article"), - "https://medium.com/article", - ).unwrap(); - database::create_archived_entry( - &conn, - &database::NewEntry { - source_identity_id: si2, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: "web".to_string(), - entity_kind: "page".to_string(), - title: Some("Resume Templates".to_string()), - visibility: "private".to_string(), - representation_kind: "html".to_string(), - source_metadata_json: "{}".to_string(), - display_metadata_json: None, - }, - ).unwrap(); - - conn - } - - #[test] - fn search_empty_query_returns_all_root_entries() { - let conn = make_test_db_with_entries(); - let all = list_root_entries(&conn).unwrap(); - let searched = search_entries(&conn, &SearchEntriesQuery::default()).unwrap(); - assert_eq!(all.len(), searched.len()); - } - - #[test] - fn search_q_filters_on_title() { - let conn = make_test_db_with_entries(); - let results = search_entries(&conn, &SearchEntriesQuery { - q: Some("polymarket".to_string()), - ..Default::default() - }).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].entity_kind, "tweet"); - } - - #[test] - fn search_entity_kind_exact_match() { - let conn = make_test_db_with_entries(); - let results = search_entries(&conn, &SearchEntriesQuery { - entity_kind: Some("page".to_string()), - ..Default::default() - }).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].source_kind, "web"); - } - - #[test] - fn search_url_like_filter() { - let conn = make_test_db_with_entries(); - let results = search_entries(&conn, &SearchEntriesQuery { - url: Some("medium.com".to_string()), - ..Default::default() - }).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].title.as_deref(), Some("Resume Templates")); - } - - #[test] - fn search_no_match_returns_empty() { - let conn = make_test_db_with_entries(); - let results = search_entries(&conn, &SearchEntriesQuery { - q: Some("zzznonexistent".to_string()), - ..Default::default() - }).unwrap(); - assert!(results.is_empty()); - } - - #[test] - fn search_multiple_filters_compound() { - let conn = make_test_db_with_entries(); - let results = search_entries(&conn, &SearchEntriesQuery { - source_kind: Some("x".to_string()), - entity_kind: Some("tweet".to_string()), - ..Default::default() - }).unwrap(); - assert_eq!(results.len(), 1); - } - - // ---- tag API tests ---- - - fn make_tag_test_db() -> (rusqlite::Connection, i64, i64) { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - database::initialize_schema(&conn).unwrap(); - let user_id = database::ensure_default_user(&conn).unwrap(); - let run = database::create_archive_run(&conn, user_id, 2).unwrap(); - (conn, user_id, run.id) - } - - fn make_entry_in_db( - conn: &rusqlite::Connection, - user_id: i64, - run_id: i64, - parent_entry_id: Option, - root_entry_id: Option, - title: &str, - url: &str, - ) -> database::ArchivedEntry { - let si = database::upsert_source_identity( - conn, "web", "page", None, Some(url), url, - ).unwrap(); - database::create_archived_entry( - conn, - &database::NewEntry { - source_identity_id: si, - archive_run_id: run_id, - parent_entry_id, - root_entry_id, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: "web".to_string(), - entity_kind: "page".to_string(), - title: Some(title.to_string()), - visibility: "private".to_string(), - representation_kind: "html".to_string(), - source_metadata_json: "{}".to_string(), - display_metadata_json: None, - }, - ).unwrap() - } - - #[test] - fn tag_tree_roots_and_children() { - let (conn, _, _) = make_tag_test_db(); - create_tag(&conn, "/science/cs").unwrap(); - create_tag(&conn, "/art").unwrap(); - - let tree = list_tag_tree(&conn).unwrap(); - assert_eq!(tree.len(), 2, "expected two root nodes"); - - let science = tree.iter().find(|n| n.tag.slug == "science").expect("science root missing"); - assert_eq!(science.children.len(), 1, "science should have one child"); - assert_eq!(science.children[0].tag.slug, "cs"); - - let art = tree.iter().find(|n| n.tag.slug == "art").expect("art root missing"); - assert!(art.children.is_empty(), "art should have no children"); - } - - #[test] - fn assign_entry_tag_is_idempotent() { - let (conn, user_id, run_id) = make_tag_test_db(); - let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Test", "https://example.com/t1"); - - assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap(); - assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap(); - - let tags = get_entry_tags(&conn, &entry.entry_uid).unwrap().unwrap(); - assert_eq!(tags.len(), 1, "idempotent assign should yield exactly one tag"); - } - - #[test] - fn remove_entry_tag_clears_assignment() { - let (conn, user_id, run_id) = make_tag_test_db(); - let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Test", "https://example.com/t2"); - - let tag = assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap().unwrap(); - remove_entry_tag(&conn, &entry.entry_uid, &tag.tag_uid).unwrap(); - - let tags = get_entry_tags(&conn, &entry.entry_uid).unwrap().unwrap(); - assert!(tags.is_empty(), "tag should be removed"); - } - - #[test] - fn entries_for_tag_includes_descendants() { - let (conn, user_id, run_id) = make_tag_test_db(); - let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Compilers Paper", "https://example.com/c1"); - - assign_entry_tag(&conn, &entry.entry_uid, "/science/cs/compilers").unwrap(); - - let results = entries_for_tag(&conn, "/science").unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].entry_uid, entry.entry_uid); - } - - #[test] - fn entries_for_tag_includes_child_entries() { - let (conn, user_id, run_id) = make_tag_test_db(); - let parent = make_entry_in_db(&conn, user_id, run_id, None, None, "Playlist", "https://example.com/pl"); - let child = make_entry_in_db( - &conn, user_id, run_id, - Some(parent.id), Some(parent.id), - "Video 1", "https://example.com/pl/v1", - ); - - assign_entry_tag(&conn, &child.entry_uid, "/science").unwrap(); - - let results = entries_for_tag(&conn, "/science").unwrap(); - assert_eq!(results.len(), 1, "only the tagged child should appear"); - assert_eq!(results[0].entry_uid, child.entry_uid); - assert_eq!(results[0].parent_entry_uid.as_deref(), Some(parent.entry_uid.as_str())); - } - - #[test] - fn search_with_tag_filter_works() { - let (conn, user_id, run_id) = make_tag_test_db(); - let entry = make_entry_in_db(&conn, user_id, run_id, None, None, "Science Article", "https://example.com/s1"); - - assign_entry_tag(&conn, &entry.entry_uid, "/science").unwrap(); - - let results = search_entries(&conn, &SearchEntriesQuery { - tag: Some("/science".to_string()), - ..Default::default() - }).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].entry_uid, entry.entry_uid); - - let empty = search_entries(&conn, &SearchEntriesQuery { - tag: Some("/art".to_string()), - ..Default::default() - }).unwrap(); - assert!(empty.is_empty(), "no entries under /art"); - } - - #[test] - fn search_without_tag_returns_roots_only() { - let (conn, user_id, run_id) = make_tag_test_db(); - let parent = make_entry_in_db(&conn, user_id, run_id, None, None, "Parent", "https://example.com/par"); - let _child = make_entry_in_db( - &conn, user_id, run_id, - Some(parent.id), Some(parent.id), - "Child", "https://example.com/par/c", - ); - - let results = search_entries(&conn, &SearchEntriesQuery::default()).unwrap(); - assert_eq!(results.len(), 1, "plain search should return root entries only"); - assert_eq!(results[0].entry_uid, parent.entry_uid); - } } diff --git a/crates/archivr-core/src/capture.rs b/crates/archivr-core/src/capture.rs deleted file mode 100644 index a1690f7..0000000 --- a/crates/archivr-core/src/capture.rs +++ /dev/null @@ -1,1261 +0,0 @@ -use anyhow::{Context, Result}; -use chrono::Local; -use serde_json::json; -use std::{ - collections::HashSet, - fs, - path::{Path, PathBuf}, -}; -use crate::{archive::ArchivePaths, database, downloader, twitter::parse_tweet_id}; - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum Source { - YouTubeVideo, - YouTubePlaylist, - YouTubeChannel, - X, - Tweet, - TweetThread, - Instagram, - Facebook, - TikTok, - Reddit, - Snapchat, - Local, - Other, -} - -#[derive(Debug, serde::Serialize)] -pub struct CaptureResult { - pub run_uid: String, - pub status: String, -} - -fn expand_shorthand_to_url(path: &str, source: &Source) -> String { - // YouTube shorthands: yt:video/ID, yt:playlist/ID, yt:@handle, yt:channel/ID, etc. - if matches!(source, Source::YouTubeVideo | Source::YouTubePlaylist | Source::YouTubeChannel) { - if let Some(after) = path.strip_prefix("yt:").or_else(|| path.strip_prefix("youtube:")) { - if let Some(id) = after - .strip_prefix("video/") - .or_else(|| after.strip_prefix("short/")) - .or_else(|| after.strip_prefix("shorts/")) - { - return format!("https://www.youtube.com/watch?v={id}"); - } - if let Some(id) = after.strip_prefix("playlist/") { - return format!("https://www.youtube.com/playlist?list={id}"); - } - if let Some(id) = after.strip_prefix("channel/") { - return format!("https://www.youtube.com/channel/{id}"); - } - if let Some(id) = after.strip_prefix("c/") { - return format!("https://www.youtube.com/c/{id}"); - } - if let Some(id) = after.strip_prefix("user/") { - return format!("https://www.youtube.com/user/{id}"); - } - if let Some(handle) = after.strip_prefix("@") { - return format!("https://www.youtube.com/@{handle}"); - } - } - } - - if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) { - if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) { - return format!("https://x.com/i/status/{tweet_id}"); - } - } - - if let Some(path) = path.strip_prefix("instagram:") { - if let Some(id) = path.strip_prefix("reel:") { - return format!("https://www.instagram.com/reel/{id}"); - } - return format!("https://www.instagram.com/{path}"); - } - if let Some(path) = path.strip_prefix("facebook:") { - return format!("https://www.facebook.com/{path}"); - } - if let Some(path) = path.strip_prefix("tiktok:") { - return format!("https://www.tiktok.com/{path}"); - } - if let Some(path) = path.strip_prefix("reddit:") { - return format!("https://www.reddit.com/{path}"); - } - if let Some(path) = path.strip_prefix("snapchat:") { - return format!("https://www.snapchat.com/{path}"); - } - - path.to_string() -} - -// INFO: yt-dlp supports a lot of sites; so, when archiving (for example) a website, the user -// -> should be asked whether they want to archive the whole website or just the video(s) on it. -fn determine_source(path: &str) -> Source { - // INFO: Extractor URLs can be found here: - // -> https://github.com/yt-dlp/yt-dlp/tree/dfc0a84c192a7357dd1768cc345d590253a14fe5/yt_dlp/extractor - // TEST: X posts can have multiple videos. - - // Shorthand schemes: yt: or youtube: - if let Some(after_scheme) = path - .strip_prefix("yt:") - .or_else(|| path.strip_prefix("youtube:")) - { - // video/ID, short/ID, shorts/ID - if after_scheme.starts_with("video/") - || after_scheme.starts_with("short/") - || after_scheme.starts_with("shorts/") - { - return Source::YouTubeVideo; - } - - // playlist/ID - if after_scheme.starts_with("playlist/") { - return Source::YouTubePlaylist; - } - - // channel/ID, c/ID, user/ID, @handle - if after_scheme.starts_with("channel/") - || after_scheme.starts_with("c/") - || after_scheme.starts_with("user/") - || after_scheme.starts_with("@") - { - return Source::YouTubeChannel; - } - } - - // Shorthand schemes: tweet:, x:, or twitter: - if let Some(after_scheme) = path - .strip_prefix("x:") - .or_else(|| path.strip_prefix("twitter:")) - .or_else(|| path.strip_prefix("tweet:")) - { - // For this scope, in comments, N is an alias for a string of type ('twitter' | 'x' | 'tweet'). - - // N:media:id - if after_scheme.starts_with("media:") - && after_scheme - .strip_prefix("media:") - .and_then(parse_tweet_id) - .is_some() - { - return Source::X; - } - - // N:tweet:id or N:x:id - if after_scheme - .strip_prefix("tweet:") - .or_else(|| after_scheme.strip_prefix("x:")) - .and_then(parse_tweet_id) - .is_some() - { - return Source::Tweet; - } - - // N:thread:id - if after_scheme - .strip_prefix("thread:") - .and_then(parse_tweet_id) - .is_some() - { - return Source::TweetThread; - } - - // N:id - if parse_tweet_id(after_scheme).is_some() { - return Source::Tweet; - } - - // N:non-id - return Source::Other; - } - - // Shorthand schemes for other yt-dlp extractors - if path.starts_with("instagram:") { - return Source::Instagram; - } - if path.starts_with("facebook:") { - return Source::Facebook; - } - if path.starts_with("tiktok:") { - return Source::TikTok; - } - if path.starts_with("reddit:") { - return Source::Reddit; - } - if path.starts_with("snapchat:") { - return Source::Snapchat; - } - - if path.starts_with("file://") { - return Source::Local; - } else if path.starts_with("http://") || path.starts_with("https://") { - // Video URLs (watch, youtu.be, shorts) - let video_re = regex::Regex::new(r"^https?://(?:www\.)?(?:youtu\.be/[0-9A-Za-z_-]+|youtube\.com/watch\?v=[0-9A-Za-z_-]+|youtube\.com/shorts/[0-9A-Za-z_-]+)") - .expect("YouTube video URL regex literal must be valid"); - if video_re.is_match(path) { - return Source::YouTubeVideo; - } - - // Playlist URLs - let playlist_re = - regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/playlist\?list=[0-9A-Za-z_-]+") - .expect("YouTube playlist URL regex literal must be valid"); - if playlist_re.is_match(path) { - return Source::YouTubePlaylist; - } - - // Channel or user URLs (channel IDs, /c/, /user/, or @handles) - let channel_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/(?:channel/[0-9A-Za-z_-]+|c/[0-9A-Za-z_-]+|user/[0-9A-Za-z_-]+|@[0-9A-Za-z_-]+)") - .expect("YouTube channel URL regex literal must be valid"); - if channel_re.is_match(path) { - return Source::YouTubeChannel; - } - - if path.starts_with("https://x.com/") { - return Source::X; - } - - if path.starts_with("https://instagram.com/") - || path.starts_with("https://www.instagram.com/") - || path.starts_with("http://instagram.com/") - || path.starts_with("http://www.instagram.com/") - { - return Source::Instagram; - } - - if path.starts_with("https://facebook.com/") - || path.starts_with("https://www.facebook.com/") - || path.starts_with("http://facebook.com/") - || path.starts_with("http://www.facebook.com/") - || path.starts_with("https://fb.watch/") - || path.starts_with("http://fb.watch/") - { - return Source::Facebook; - } - - if path.starts_with("https://tiktok.com/") - || path.starts_with("https://www.tiktok.com/") - || path.starts_with("http://tiktok.com/") - || path.starts_with("http://www.tiktok.com/") - { - return Source::TikTok; - } - - if path.starts_with("https://reddit.com/") - || path.starts_with("https://www.reddit.com/") - || path.starts_with("http://reddit.com/") - || path.starts_with("http://www.reddit.com/") - || path.starts_with("https://redd.it/") - || path.starts_with("http://redd.it/") - { - return Source::Reddit; - } - - if path.starts_with("https://snapchat.com/") - || path.starts_with("https://www.snapchat.com/") - || path.starts_with("http://snapchat.com/") - || path.starts_with("http://www.snapchat.com/") - { - return Source::Snapchat; - } - } - if Path::new(path).exists() { - return Source::Local; - } - Source::Other -} - -fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result { - let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); - - println!("Checking {}", path.display()); - - Ok(path.exists()) -} - -fn move_temp_to_raw(file: &Path, hash: &str, store_path: &Path) -> Result<()> { - let file_extension = file - .extension() - .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())); - let raw_relpath = raw_relative_path_from_hash(hash, &file_extension)?; - let destination = store_path.join(raw_relpath); - - if let Some(parent) = destination.parent() { - fs::create_dir_all(parent)?; - } - - fs::rename(file, destination)?; - - Ok(()) -} - -fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result { - let mut chars = hash.chars(); - let first_letter = chars.next().context("hash must not be empty")?; - let second_letter = chars - .next() - .context("hash must be at least two characters")?; - - Ok(PathBuf::from("raw") - .join(first_letter.to_string()) - .join(second_letter.to_string()) - .join(format!("{hash}{file_extension}"))) -} - -fn path_to_store_string(path: &Path) -> String { - path.to_string_lossy().replace('\\', "/") -} - -fn extension_without_dot(file_extension: &str) -> Option { - file_extension - .strip_prefix('.') - .filter(|extension| !extension.is_empty()) - .map(|extension| extension.to_string()) -} - -fn blob_record_for_raw_relpath( - store_path: &Path, - raw_relpath: &Path, -) -> Result { - let absolute_path = store_path.join(raw_relpath); - let file_name = raw_relpath - .file_name() - .and_then(|name| name.to_str()) - .context("raw artifact path must have a UTF-8 file name")?; - let (sha256, extension) = match file_name.rsplit_once('.') { - Some((hash, extension)) => (hash.to_string(), Some(extension.to_string())), - None => (file_name.to_string(), None), - }; - - Ok(database::BlobRecord { - sha256, - byte_size: fs::metadata(&absolute_path) - .with_context(|| format!("failed to stat raw artifact {}", absolute_path.display()))? - .len() as i64, - mime_type: None, - extension, - raw_relpath: path_to_store_string(raw_relpath), - }) -} - -fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str) { - match source { - Source::YouTubeVideo => ("youtube", "video", "video"), - Source::YouTubePlaylist => ("youtube", "playlist", "container"), - Source::YouTubeChannel => ("youtube", "channel", "container"), - Source::X => ("x", "post", "video"), - Source::Tweet => ("x", "tweet", "tweet_json"), - Source::TweetThread => ("x", "tweet_thread", "tweet_json"), - Source::Instagram => ("instagram", "post", "video"), - Source::Facebook => ("facebook", "post", "video"), - Source::TikTok => ("tiktok", "video", "video"), - Source::Reddit => ("reddit", "post", "video"), - Source::Snapchat => ("snapchat", "story", "video"), - Source::Local => ("local", "file", "file"), - Source::Other => ("other", "unknown", "unknown"), - } -} - -fn local_file_extension(path: &str) -> String { - Path::new(path.trim_start_matches("file://")) - .extension() - .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) -} - -fn media_file_extension(source: Source, path: &str) -> String { - match source { - Source::YouTubeVideo - | Source::X - | Source::Instagram - | Source::Facebook - | Source::TikTok - | Source::Reddit - | Source::Snapchat => ".mp4".to_string(), - Source::Local => local_file_extension(path), - _ => String::new(), - } -} - -fn tweet_id_from_archive_path(path: &str) -> Option { - path.split(':').next_back().and_then(parse_tweet_id) -} - -fn create_structured_root(store_path: &Path, entry: &database::ArchivedEntry) -> Result<()> { - debug_assert!(entry.entry_uid.starts_with("entry_")); - fs::create_dir_all(store_path.join(&entry.structured_root_relpath))?; - Ok(()) -} - -fn record_media_entry( - conn: &rusqlite::Connection, - store_path: &Path, - user_id: i64, - run: &database::ArchiveRun, - item: &database::ArchiveRunItem, - requested_locator: &str, - canonical_locator: &str, - source: Source, - hash: &str, - file_extension: &str, - byte_size: i64, -) -> Result { - debug_assert!(run.run_uid.starts_with("run_")); - debug_assert!(item.item_uid.starts_with("item_")); - let (source_kind, entity_kind, representation_kind) = source_metadata(source); - let raw_relpath = raw_relative_path_from_hash(hash, file_extension)?; - let blob = database::BlobRecord { - sha256: hash.to_string(), - byte_size, - mime_type: None, - extension: extension_without_dot(file_extension), - raw_relpath: path_to_store_string(&raw_relpath), - }; - let blob_id = database::upsert_blob(conn, &blob)?; - let source_identity_id = database::upsert_source_identity( - conn, - source_kind, - entity_kind, - None, - Some(canonical_locator), - canonical_locator, - )?; - let entry = database::create_archived_entry( - conn, - &database::NewEntry { - source_identity_id, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: source_kind.to_string(), - entity_kind: entity_kind.to_string(), - title: None, - visibility: "private".to_string(), - representation_kind: representation_kind.to_string(), - source_metadata_json: json!({ - "requested_locator": requested_locator, - "canonical_locator": canonical_locator - }) - .to_string(), - display_metadata_json: None, - }, - )?; - create_structured_root(store_path, &entry)?; - database::add_entry_artifact( - conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: "primary_media".to_string(), - storage_area: "raw".to_string(), - relpath: blob.raw_relpath, - blob_id: Some(blob_id), - logical_path: None, - metadata_json: None, - }, - )?; - database::complete_archive_run_item(conn, item.id, entry.id)?; - Ok(entry) -} - -fn record_tweet_entry( - conn: &rusqlite::Connection, - store_path: &Path, - user_id: i64, - run: &database::ArchiveRun, - item: &database::ArchiveRunItem, - requested_locator: &str, - source: Source, - tweet_id: &str, -) -> Result { - debug_assert!(run.run_uid.starts_with("run_")); - debug_assert!(item.item_uid.starts_with("item_")); - let (source_kind, entity_kind, representation_kind) = source_metadata(source); - let canonical_locator = format!("https://x.com/i/status/{tweet_id}"); - let source_identity_id = database::upsert_source_identity( - conn, - source_kind, - entity_kind, - Some(tweet_id), - Some(&canonical_locator), - &canonical_locator, - )?; - let entry = database::create_archived_entry( - conn, - &database::NewEntry { - source_identity_id, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: source_kind.to_string(), - entity_kind: entity_kind.to_string(), - title: None, - visibility: "private".to_string(), - representation_kind: representation_kind.to_string(), - source_metadata_json: json!({ - "tweet_id": tweet_id, - "requested_locator": requested_locator - }) - .to_string(), - display_metadata_json: None, - }, - )?; - create_structured_root(store_path, &entry)?; - - let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); - database::add_entry_artifact( - conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: "raw_tweet_json".to_string(), - storage_area: "raw_tweets".to_string(), - relpath: path_to_store_string(&tweet_json_relpath), - blob_id: None, - logical_path: None, - metadata_json: None, - }, - )?; - - let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; - for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? { - let raw_path = PathBuf::from(&raw_relpath); - let blob = blob_record_for_raw_relpath(store_path, &raw_path)?; - let blob_id = database::upsert_blob(conn, &blob)?; - database::add_entry_artifact( - conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: role, - storage_area: "raw".to_string(), - relpath: raw_relpath, - blob_id: Some(blob_id), - logical_path: None, - metadata_json: None, - }, - )?; - } - - database::complete_archive_run_item(conn, item.id, entry.id)?; - Ok(entry) -} - -fn tweet_raw_artifacts(tweet_json: &str) -> Result> { - let regex = regex::Regex::new(r#""(avatar_local_path|local_path)": "([^"\n]+)""#)?; - let mut seen = HashSet::new(); - let mut artifacts = Vec::new(); - - for captures in regex.captures_iter(tweet_json) { - let relpath = captures[2].to_string(); - if !relpath.starts_with("raw/") || !seen.insert(relpath.clone()) { - continue; - } - - let role = if &captures[1] == "avatar_local_path" { - "avatar" - } else { - "media" - }; - artifacts.push((role.to_string(), relpath)); - } - - Ok(artifacts) -} - -/// Marks the run and item as failed in the database, returns the error. -/// Call sites: `return Err(fail_run(&conn, &run, &item, "message"));` -fn fail_run( - conn: &rusqlite::Connection, - run: &database::ArchiveRun, - item: &database::ArchiveRunItem, - message: &str, -) -> anyhow::Error { - let _ = database::fail_archive_run_item(conn, item.id, message); - let _ = database::fail_archive_run(conn, run.id, message); - anyhow::anyhow!("{}", message) -} - -pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result { - let timestamp = Local::now().format("%Y-%m-%dT%H-%M-%S%.3f").to_string(); - let store_path = &archive_paths.store_path; - - let conn = database::open_or_initialize(&archive_paths.archive_path)?; - let user_id = database::ensure_default_user(&conn)?; - - let source = determine_source(locator); - let (source_kind, entity_kind, _) = source_metadata(source); - - let run = database::create_archive_run(&conn, user_id, 1)?; - let item = database::create_archive_run_item( - &conn, - run.id, - None, - 0, - locator, - None, - source_kind, - entity_kind, - )?; - - // Sources: Other (not yet implemented) - if source == Source::Other { - return Err(fail_run( - &conn, - &run, - &item, - "Archiving from this source is not yet implemented.", - )); - } - - // Sources: Tweets or Twitter Threads - if matches!(source, Source::Tweet | Source::TweetThread) { - let tweet_id = match tweet_id_from_archive_path(locator) { - Some(tweet_id) => tweet_id, - None => { - return Err(fail_run( - &conn, - &run, - &item, - "Failed to archive tweet: invalid tweet ID", - )); - } - }; - - match downloader::tweets::archive( - locator, - source == Source::TweetThread, - store_path, - ×tamp, - ) { - Ok(_) => { - record_tweet_entry( - &conn, - store_path, - user_id, - &run, - &item, - locator, - source, - &tweet_id, - )?; - database::finish_archive_run(&conn, run.id)?; - return Ok(CaptureResult { - run_uid: run.run_uid.clone(), - status: "completed".to_string(), - }); - } - Err(e) => { - return Err(fail_run( - &conn, - &run, - &item, - &format!("Failed to archive tweet: {e}"), - )); - } - } - } - - // Sources, for which yt-dlp is needed - let requested_locator = locator.to_string(); - let path = expand_shorthand_to_url(locator, &source); - let hash = match source { - Source::YouTubeVideo - | Source::X - | Source::Instagram - | Source::Facebook - | Source::TikTok - | Source::Reddit - | Source::Snapchat => { - match downloader::ytdlp::download(path.clone(), store_path, ×tamp) { - Ok(h) => h, - Err(e) => { - return Err(fail_run( - &conn, - &run, - &item, - &format!("Failed to download media: {e}"), - )); - } - } - } - Source::Local => { - match downloader::local::save(path.clone(), store_path, ×tamp) { - Ok(h) => h, - Err(e) => { - return Err(fail_run( - &conn, - &run, - &item, - &format!("Failed to archive local file: {e}"), - )); - } - } - } - Source::YouTubePlaylist | Source::YouTubeChannel => { - return Err(fail_run( - &conn, - &run, - &item, - "Playlist and channel container expansion are not yet implemented.", - )); - } - _ => unreachable!(), - }; - - let file_extension = media_file_extension(source, &path); - let temp_file = store_path - .join("temp") - .join(×tamp) - .join(format!("{timestamp}{file_extension}")); - let byte_size = fs::metadata(&temp_file) - .with_context(|| format!("failed to stat staged file {}", temp_file.display()))? - .len() as i64; - - let hash_exists = hash_exists(&hash, &file_extension, store_path)?; - - if hash_exists { - let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); - } else { - move_temp_to_raw( - &store_path - .join("temp") - .join(×tamp) - .join(format!("{timestamp}{file_extension}")), - &hash, - store_path, - )?; - let _ = fs::remove_dir_all(store_path.join("temp").join(×tamp)); - } - - record_media_entry( - &conn, - store_path, - user_id, - &run, - &item, - &requested_locator, - &path, - source, - &hash, - &file_extension, - byte_size, - )?; - database::finish_archive_run(&conn, run.id)?; - - Ok(CaptureResult { - run_uid: run.run_uid.clone(), - status: "completed".to_string(), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{archive, database}; - use chrono::Local; - use std::{env, fs}; - - struct TestCase<'a> { - url: &'a str, - expected: Source, - } - - #[test] - fn test_tweet_sources() { - let cases = [ - TestCase { - url: "tweet:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "x:tweet:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "x:x:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "twitter:x:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "twitter:tweet:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "tweet:media:1234567890", - expected: Source::X, - }, - TestCase { - url: "x:media:1234567890", - expected: Source::X, - }, - TestCase { - url: "x:thread:1234567890", - expected: Source::TweetThread, - }, - TestCase { - url: "twitter:thread:1234567890", - expected: Source::TweetThread, - }, - TestCase { - url: "tweet:thread:1234567890", - expected: Source::TweetThread, - }, - TestCase { - url: "tweet:not-a-number", - expected: Source::Other, - }, - TestCase { - url: "tweet:media:not-a-number", - expected: Source::Other, - }, - TestCase { - url: "x:media:not-a-number", - expected: Source::Other, - }, - ]; - - for case in &cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_resolve_source_path() { - assert_eq!( - expand_shorthand_to_url("tweet:media:1234567890", &Source::X), - "https://x.com/i/status/1234567890" - ); - assert_eq!( - expand_shorthand_to_url("instagram:reel/ABC123", &Source::Instagram), - "https://www.instagram.com/reel/ABC123" - ); - assert_eq!( - expand_shorthand_to_url("facebook:watch?v=123456", &Source::Facebook), - "https://www.facebook.com/watch?v=123456" - ); - assert_eq!( - expand_shorthand_to_url("tiktok:@someone/video/123456789", &Source::TikTok), - "https://www.tiktok.com/@someone/video/123456789" - ); - assert_eq!( - expand_shorthand_to_url("reddit:r/videos/comments/abc123/example", &Source::Reddit), - "https://www.reddit.com/r/videos/comments/abc123/example" - ); - assert_eq!( - expand_shorthand_to_url("snapchat:discover/some-story/1234567890", &Source::Snapchat), - "https://www.snapchat.com/discover/some-story/1234567890" - ); - assert_eq!( - expand_shorthand_to_url("tweet:1234567890", &Source::Tweet), - "tweet:1234567890" - ); - // YouTube shorthands must expand to full URLs before yt-dlp sees them - assert_eq!( - expand_shorthand_to_url("yt:video/MntbN1DdEP0", &Source::YouTubeVideo), - "https://www.youtube.com/watch?v=MntbN1DdEP0" - ); - assert_eq!( - expand_shorthand_to_url("yt:shorts/EtC99eWiwRI", &Source::YouTubeVideo), - "https://www.youtube.com/watch?v=EtC99eWiwRI" - ); - assert_eq!( - expand_shorthand_to_url("youtube:video/UHxw-L2WyyY", &Source::YouTubeVideo), - "https://www.youtube.com/watch?v=UHxw-L2WyyY" - ); - assert_eq!( - expand_shorthand_to_url("yt:playlist/PL9vTTBa7QaQO", &Source::YouTubePlaylist), - "https://www.youtube.com/playlist?list=PL9vTTBa7QaQO" - ); - assert_eq!( - expand_shorthand_to_url("yt:@CoreDumpped", &Source::YouTubeChannel), - "https://www.youtube.com/@CoreDumpped" - ); - assert_eq!( - expand_shorthand_to_url("yt:channel/UCxyz123", &Source::YouTubeChannel), - "https://www.youtube.com/channel/UCxyz123" - ); - // Full YouTube URLs pass through unchanged - assert_eq!( - expand_shorthand_to_url("https://www.youtube.com/watch?v=UHxw-L2WyyY", &Source::YouTubeVideo), - "https://www.youtube.com/watch?v=UHxw-L2WyyY" - ); - } - - #[test] - fn test_youtube_sources() { - // --- YouTube Video URLs --- - let video_cases = [ - TestCase { - url: "https://www.youtube.com/watch?v=UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "https://youtu.be/UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "https://www.youtube.com/shorts/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - ]; - - for case in &video_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - - // --- YouTube Playlist URLs --- - let playlist_cases = [TestCase { - url: "https://www.youtube.com/playlist?list=PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", - expected: Source::YouTubePlaylist, - }]; - - for case in &playlist_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - - // --- YouTube Channel URLs --- - let channel_cases = [ - TestCase { - url: "https://www.youtube.com/channel/CoreDumpped", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://www.youtube.com/@CoreDumpped", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://www.youtube.com/c/YouTubeCreators", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://www.youtube.com/user/pewdiepie", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "https://youtube.com/@pewdiepie?si=KOcLN_KPYNpe5f_8", - expected: Source::YouTubeChannel, - }, - ]; - - for case in &channel_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - - // --- Shorthand scheme URLs --- - let shorthand_cases = [ - // Videos - TestCase { - url: "yt:video/UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "youtube:video/UHxw-L2WyyY", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "yt:short/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "yt:shorts/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - TestCase { - url: "youtube:shorts/EtC99eWiwRI", - expected: Source::YouTubeVideo, - }, - // Playlists - TestCase { - url: "yt:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", - expected: Source::YouTubePlaylist, - }, - TestCase { - url: "youtube:playlist/PL9vTTBa7QaQOoMfpP3ztvgyQkPWDPfJez", - expected: Source::YouTubePlaylist, - }, - // Channels - TestCase { - url: "yt:channel/UCxyz123", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "yt:c/YouTubeCreators", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "yt:user/pewdiepie", - expected: Source::YouTubeChannel, - }, - TestCase { - url: "youtube:@CoreDumpped", - expected: Source::YouTubeChannel, - }, - ]; - - for case in &shorthand_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_x_sources() { - let x_cases = [ - TestCase { - url: "https://x.com/some_post", - expected: Source::X, - }, - TestCase { - url: "x:1234567890", - expected: Source::Tweet, - }, - TestCase { - url: "twitter:1234567890", - expected: Source::Tweet, - }, - ]; - - for case in &x_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_other_social_sources() { - let social_cases = [ - TestCase { - url: "https://www.instagram.com/reel/ABC123/", - expected: Source::Instagram, - }, - TestCase { - url: "instagram:reel/ABC123", - expected: Source::Instagram, - }, - TestCase { - url: "https://www.facebook.com/watch/?v=123456", - expected: Source::Facebook, - }, - TestCase { - url: "facebook:watch?v=123456", - expected: Source::Facebook, - }, - TestCase { - url: "https://www.tiktok.com/@someone/video/123456789", - expected: Source::TikTok, - }, - TestCase { - url: "tiktok:@someone/video/123456789", - expected: Source::TikTok, - }, - TestCase { - url: "https://www.reddit.com/r/videos/comments/abc123/example/", - expected: Source::Reddit, - }, - TestCase { - url: "reddit:r/videos/comments/abc123/example", - expected: Source::Reddit, - }, - TestCase { - url: "https://www.snapchat.com/discover/some-story/1234567890", - expected: Source::Snapchat, - }, - TestCase { - url: "snapchat:discover/some-story/1234567890", - expected: Source::Snapchat, - }, - ]; - - for case in &social_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_non_youtube_sources() { - let other_cases = [ - TestCase { - url: "file:///local/path/file.mp4", - expected: Source::Local, - }, - TestCase { - url: "https://example.com/", - expected: Source::Other, - }, - TestCase { - url: "https://example.com/?redirect=instagram.com/reel/ABC123", - expected: Source::Other, - }, - TestCase { - url: "https://notfacebook.com/watch?v=123456", - expected: Source::Other, - }, - ]; - - for case in &other_cases { - assert_eq!( - determine_source(case.url), - case.expected, - "Failed for URL: {}", - case.url - ); - } - } - - #[test] - fn test_existing_local_path_source() { - let path = env::current_dir().unwrap().join("Cargo.toml"); - assert_eq!( - determine_source(path.to_str().unwrap()), - Source::Local, - "existing filesystem paths should be archived as local files" - ); - } - - #[test] - fn test_initialize_store_directories() { - let store_path = env::temp_dir().join(format!( - "archivr-test-{}", - Local::now().format("%Y%m%d%H%M%S%3f") - )); - - archive::initialize_store_directories(&store_path).unwrap(); - - assert!(store_path.join("raw").is_dir()); - assert!(store_path.join("raw_tweets").is_dir()); - assert!(store_path.join("structured").is_dir()); - assert!(store_path.join("temp").is_dir()); - assert!(!store_path.join("tmp").exists()); - - fs::remove_dir_all(store_path).unwrap(); - } - - #[test] - fn test_record_tweet_entry_links_json_and_raw_artifacts() { - let store_path = env::temp_dir().join(format!( - "archivr-tweet-db-test-{}", - Local::now().format("%Y%m%d%H%M%S%3f") - )); - let _ = fs::remove_dir_all(&store_path); - archive::initialize_store_directories(&store_path).unwrap(); - fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap(); - fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap(); - fs::write( - store_path - .join("raw") - .join("a") - .join("b") - .join("abcdef.jpg"), - b"avatar", - ) - .unwrap(); - fs::write( - store_path - .join("raw") - .join("c") - .join("d") - .join("cdef01.mp4"), - b"media", - ) - .unwrap(); - fs::write( - store_path.join("raw_tweets").join("tweet-123.json"), - r#"{ - "author": { "avatar_local_path": "raw/a/b/abcdef.jpg" }, - "entities": { "media": [{ "local_path": "raw/c/d/cdef01.mp4" }] } -}"#, - ) - .unwrap(); - - let conn = rusqlite::Connection::open_in_memory().unwrap(); - database::initialize_schema(&conn).unwrap(); - let user_id = database::ensure_default_user(&conn).unwrap(); - let run = database::create_archive_run(&conn, user_id, 1).unwrap(); - let item = database::create_archive_run_item( - &conn, - run.id, - None, - 0, - "tweet:123", - None, - "x", - "tweet", - ) - .unwrap(); - - let entry = record_tweet_entry( - &conn, - &store_path, - user_id, - &run, - &item, - "tweet:123", - Source::Tweet, - "123", - ) - .unwrap(); - database::finish_archive_run(&conn, run.id).unwrap(); - - let artifact_count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM entry_artifacts WHERE entry_id = ?1", - [entry.id], - |row| row.get(0), - ) - .unwrap(); - let blob_count: i64 = conn - .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get(0)) - .unwrap(); - let run_status: String = conn - .query_row( - "SELECT status FROM archive_runs WHERE id = ?1", - [run.id], - |row| row.get(0), - ) - .unwrap(); - - assert_eq!(artifact_count, 3); - assert_eq!(blob_count, 2); - assert_eq!(run_status, "completed"); - assert!(store_path.join(&entry.structured_root_relpath).is_dir()); - - let _ = fs::remove_dir_all(store_path); - } -} diff --git a/crates/archivr-core/src/database.rs b/crates/archivr-core/src/database.rs index d207c35..5ec3f74 100644 --- a/crates/archivr-core/src/database.rs +++ b/crates/archivr-core/src/database.rs @@ -63,16 +63,6 @@ pub struct NewArtifact { pub metadata_json: Option, } -#[derive(Debug, Clone)] -pub struct TagRecord { - pub id: i64, - pub tag_uid: String, - pub parent_tag_id: Option, - pub name: String, - pub slug: String, - pub full_path: String, -} - pub fn database_path(archive_path: &Path) -> PathBuf { archive_path.join(DATABASE_FILE_NAME) } @@ -224,7 +214,6 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> { CREATE INDEX IF NOT EXISTS idx_entry_artifacts_entry_id ON entry_artifacts(entry_id); CREATE INDEX IF NOT EXISTS idx_entry_artifacts_blob_id ON entry_artifacts(blob_id); CREATE INDEX IF NOT EXISTS idx_tags_parent_tag_id ON tags(parent_tag_id); - CREATE INDEX IF NOT EXISTS idx_entry_tag_assignments_tag_id ON entry_tag_assignments(tag_id); "#, )?; Ok(()) @@ -499,104 +488,6 @@ pub fn add_entry_artifact(conn: &Connection, artifact: &NewArtifact) -> Result Result<()> { - conn.execute( - "DELETE FROM entry_tag_assignments WHERE entry_id = ?1 AND tag_id = ?2", - params![entry_id, tag_id], - )?; - Ok(()) -} - -pub fn list_all_tags(conn: &Connection) -> Result> { - let mut stmt = conn.prepare( - "SELECT id, tag_uid, parent_tag_id, name, slug, full_path - FROM tags - ORDER BY full_path", - )?; - let records = stmt - .query_map([], |row| { - Ok(TagRecord { - id: row.get(0)?, - tag_uid: row.get(1)?, - parent_tag_id: row.get(2)?, - name: row.get(3)?, - slug: row.get(4)?, - full_path: row.get(5)?, - }) - })? - .collect::, _>>() - .context("failed to list tags")?; - Ok(records) -} - -pub fn list_tags_for_entry(conn: &Connection, entry_id: i64) -> Result> { - let mut stmt = conn.prepare( - "SELECT t.id, t.tag_uid, t.parent_tag_id, t.name, t.slug, t.full_path - FROM tags t - JOIN entry_tag_assignments eta ON eta.tag_id = t.id - WHERE eta.entry_id = ?1 - ORDER BY t.full_path", - )?; - let records = stmt - .query_map([entry_id], |row| { - Ok(TagRecord { - id: row.get(0)?, - tag_uid: row.get(1)?, - parent_tag_id: row.get(2)?, - name: row.get(3)?, - slug: row.get(4)?, - full_path: row.get(5)?, - }) - })? - .collect::, _>>() - .context("failed to list tags for entry")?; - Ok(records) -} - -pub fn get_tag_by_uid(conn: &Connection, tag_uid: &str) -> Result> { - conn.query_row( - "SELECT id, tag_uid, parent_tag_id, name, slug, full_path - FROM tags WHERE tag_uid = ?1", - [tag_uid], - |row| { - Ok(TagRecord { - id: row.get(0)?, - tag_uid: row.get(1)?, - parent_tag_id: row.get(2)?, - name: row.get(3)?, - slug: row.get(4)?, - full_path: row.get(5)?, - }) - }, - ) - .optional() - .context("failed to get tag by uid") -} - -pub fn get_tag_by_path(conn: &Connection, full_path: &str) -> Result> { - conn.query_row( - "SELECT id, tag_uid, parent_tag_id, name, slug, full_path - FROM tags WHERE full_path = ?1", - [full_path], - |row| { - Ok(TagRecord { - id: row.get(0)?, - tag_uid: row.get(1)?, - parent_tag_id: row.get(2)?, - name: row.get(3)?, - slug: row.get(4)?, - full_path: row.get(5)?, - }) - }, - ) - .optional() - .context("failed to get tag by path") -} - #[cfg(test)] pub fn set_public_settings( conn: &Connection, @@ -644,6 +535,7 @@ pub fn main_archive_entry_count(conn: &Connection) -> Result { Ok(count) } +#[cfg(test)] pub fn create_tag_path(conn: &Connection, full_path: &str) -> Result { let segments = normalized_tag_segments(full_path)?; let mut parent_tag_id = None; @@ -685,6 +577,7 @@ pub fn create_tag_path(conn: &Connection, full_path: &str) -> Result { Ok(current_id) } +#[cfg(test)] pub fn assign_entry_to_tag(conn: &Connection, entry_id: i64, tag_id: i64) -> Result<()> { conn.execute( "INSERT OR IGNORE INTO entry_tag_assignments (entry_id, tag_id) @@ -694,6 +587,7 @@ pub fn assign_entry_to_tag(conn: &Connection, entry_id: i64, tag_id: i64) -> Res Ok(()) } +#[cfg(test)] pub fn entry_count_for_tag_path(conn: &Connection, full_path: &str) -> Result { let count = conn.query_row( "WITH RECURSIVE descendants(id) AS ( @@ -759,6 +653,7 @@ fn validate_visibility(visibility: &str) -> Result<()> { } } +#[cfg(test)] fn normalized_tag_segments(full_path: &str) -> Result> { let segments = full_path .trim() @@ -774,6 +669,7 @@ fn normalized_tag_segments(full_path: &str) -> Result> { Ok(segments) } +#[cfg(test)] fn humanize_slug(slug: &str) -> String { slug.split('-') .map(|part| { diff --git a/crates/archivr-core/src/lib.rs b/crates/archivr-core/src/lib.rs index cd97ffb..f1114c1 100644 --- a/crates/archivr-core/src/lib.rs +++ b/crates/archivr-core/src/lib.rs @@ -1,4 +1,3 @@ -pub mod capture; pub mod archive; pub mod database; pub mod downloader; diff --git a/crates/archivr-server/Cargo.toml b/crates/archivr-server/Cargo.toml index 4c3548e..b90f678 100644 --- a/crates/archivr-server/Cargo.toml +++ b/crates/archivr-server/Cargo.toml @@ -10,10 +10,8 @@ axum.workspace = true serde.workspace = true tokio.workspace = true toml.workspace = true -tower.workspace = true tower-http.workspace = true [dev-dependencies] tempfile.workspace = true tower.workspace = true -serde_json.workspace = true diff --git a/crates/archivr-server/src/main.rs b/crates/archivr-server/src/main.rs index 47f9532..13eb826 100644 --- a/crates/archivr-server/src/main.rs +++ b/crates/archivr-server/src/main.rs @@ -1,11 +1,9 @@ mod registry; mod routes; -use anyhow::{Context, Result}; +use anyhow::Result; use std::{net::SocketAddr, path::PathBuf}; -const DEFAULT_BIND: &str = "127.0.0.1:8080"; - #[tokio::main] async fn main() -> Result<()> { let config_path = std::env::args() @@ -13,27 +11,8 @@ async fn main() -> Result<()> { .map(PathBuf::from) .unwrap_or_else(|| PathBuf::from("archivr-server.toml")); let registry = registry::load_registry(&config_path)?; - let app = routes::app(registry.clone()); - - // Bind address priority: ARCHIVR_BIND env var > TOML bind field > default loopback. - let bind_str = std::env::var("ARCHIVR_BIND") - .ok() - .or_else(|| registry.bind.clone()) - .unwrap_or_else(|| DEFAULT_BIND.to_string()); - - let addr: SocketAddr = bind_str - .parse() - .with_context(|| format!("invalid bind address: {bind_str}"))?; - - // Warn when the server is reachable beyond localhost — it has no authentication. - if !addr.ip().is_loopback() { - eprintln!( - "warn: archivr-server is bound to {addr} — \ - this server has no authentication. \ - Only expose it on a trusted network." - ); - } - + let app = routes::app(registry); + let addr = SocketAddr::from(([127, 0, 0, 1], 8080)); let listener = tokio::net::TcpListener::bind(addr).await?; println!("archivr-server listening on http://{addr}"); axum::serve(listener, app).await?; diff --git a/crates/archivr-server/src/registry.rs b/crates/archivr-server/src/registry.rs index a9fef6c..c186d01 100644 --- a/crates/archivr-server/src/registry.rs +++ b/crates/archivr-server/src/registry.rs @@ -14,12 +14,7 @@ pub struct MountedArchive { #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct ServerRegistry { - #[serde(default)] pub archives: Vec, - /// Optional bind address for the server. Defaults to `127.0.0.1:8080`. - /// Set this to `0.0.0.0:8080` only on trusted networks — the server has no authentication. - #[serde(default)] - pub bind: Option, } pub fn load_registry(path: &Path) -> Result { @@ -83,7 +78,6 @@ mod tests { label: "Personal".to_string(), archive_path: archive_path.clone(), }], - bind: None, }; let path = temp.path().join("server.toml"); save_registry(&path, ®istry).unwrap(); @@ -108,36 +102,10 @@ mod tests { archive_path: PathBuf::from("/tmp/b/.archivr"), }, ], - bind: None, }; let err = validate_registry(®istry).unwrap_err().to_string(); assert!(err.contains("duplicate archive id")); } - - #[test] - fn registry_bind_field_round_trips() { - let toml = r#"bind = "127.0.0.1:9090""#; - let registry: ServerRegistry = toml::from_str(toml).unwrap(); - assert_eq!(registry.bind.as_deref(), Some("127.0.0.1:9090")); - assert!(registry.archives.is_empty()); - } - - #[test] - fn registry_bind_field_defaults_to_none_when_absent() { - let toml = r#""#; - let registry: ServerRegistry = toml::from_str(toml).unwrap(); - assert!(registry.bind.is_none()); - } - - #[test] - fn registry_bind_field_does_not_affect_archive_validation() { - let registry = ServerRegistry { - archives: vec![], - bind: Some("0.0.0.0:8080".to_string()), - }; - // validate_registry does not reject non-loopback bind — that's main's concern. - assert!(validate_registry(®istry).is_ok()); - } } diff --git a/crates/archivr-server/src/routes.rs b/crates/archivr-server/src/routes.rs index 014544a..dff563b 100644 --- a/crates/archivr-server/src/routes.rs +++ b/crates/archivr-server/src/routes.rs @@ -1,39 +1,14 @@ -// ── Security Boundary ──────────────────────────────────────────────────────── -// All routes are currently trusted-local: no authentication or authorization -// middleware is applied. The server is designed to bind on 127.0.0.1 only. -// -// Route classification (for when middleware is added later): -// -// STATIC — safe to expose publicly: GET / and static /assets/* -// READ — safe to expose read-only: GET /health -// GET /api/archives -// GET /api/archives/:id/entries -// GET /api/archives/:id/entries/search -// GET /api/archives/:id/entries/:uid -// GET /api/archives/:id/entries/:uid/artifacts/:idx -// GET /api/archives/:id/runs -// GET /api/archives/:id/tags -// ADMIN — requires auth if ever public: GET /api/admin/archives -// WRITE — requires auth if ever public: POST /api/archives/:id/captures -// POST /api/archives/:id/tags -// PUT /api/archives/:id/tags/:tag_id -// DELETE /api/archives/:id/tags/:tag_id -// -// Do not add middleware here until the auth model is chosen. See docs/README.md. -// ───────────────────────────────────────────────────────────────────────────── - use std::{path::PathBuf, sync::Arc}; -use archivr_core::{archive, capture, database}; +use archivr_core::{archive, database}; use axum::{ Json, Router, - extract::{Path, Query, Request, State}, + extract::{Path, State}, http::StatusCode, response::{IntoResponse, Response}, - routing::{delete, get, post}, + routing::get, }; use tower_http::services::{ServeDir, ServeFile}; -use tower::ServiceExt; use crate::registry::{MountedArchive, ServerRegistry}; @@ -42,12 +17,6 @@ pub struct AppState { registry: Arc, } -#[derive(Debug, serde::Deserialize, Default)] -pub struct EntrySearchParams { - pub q: Option, - pub tag: Option, -} - pub fn app(registry: ServerRegistry) -> Router { let state = AppState { registry: Arc::new(registry), @@ -58,26 +27,11 @@ pub fn app(registry: ServerRegistry) -> Router { .route("/health", get(|| async { "ok" })) .route("/api/archives", get(list_archives)) .route("/api/archives/:archive_id/entries", get(list_entries)) - .route("/api/archives/:archive_id/entries/search", get(search_entries_handler)) .route( "/api/archives/:archive_id/entries/:entry_uid", get(entry_detail), ) - .route( - "/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index", - get(serve_artifact), - ) .route("/api/archives/:archive_id/runs", get(list_runs)) - .route("/api/archives/:archive_id/captures", post(capture_handler)) - .route("/api/archives/:archive_id/tags", get(list_tags).post(create_tag_handler)) - .route( - "/api/archives/:archive_id/entries/:entry_uid/tags", - get(list_entry_tags).post(assign_entry_tag_handler), - ) - .route( - "/api/archives/:archive_id/entries/:entry_uid/tags/:tag_uid", - delete(remove_entry_tag_handler), - ) .nest_service("/assets", ServeDir::new(&static_dir)) .fallback_service(ServeFile::new(static_dir.join("index.html"))) .with_state(state) @@ -102,22 +56,6 @@ async fn list_entries( Ok(Json(archive::list_root_entries(&conn)?)) } -async fn search_entries_handler( - State(state): State, - Path(archive_id): Path, - Query(params): Query, -) -> Result>, ApiError> { - let mounted = mounted_archive(&state, &archive_id)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - let raw = params.q.as_deref().unwrap_or(""); - let mut search_query = archive::parse_search_query(raw) - .map_err(|prefix| ApiError::bad_request(&format!("unknown search prefix: {prefix}")))?; - if let Some(tag) = params.tag { - search_query.tag = Some(tag); - } - Ok(Json(archive::search_entries(&conn, &search_query)?)) -} - async fn entry_detail( State(state): State, Path((archive_id, entry_uid)): Path<(String, String)>, @@ -138,125 +76,6 @@ async fn list_runs( Ok(Json(archive::list_runs(&conn)?)) } -async fn serve_artifact( - State(state): State, - Path((archive_id, entry_uid, artifact_index)): Path<(String, String, usize)>, - req: Request, -) -> Result { - let mounted = mounted_archive(&state, &archive_id)?; - let paths = archive::read_archive_paths(&mounted.archive_path)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - let detail = archive::get_entry_detail(&conn, &entry_uid)? - .ok_or(ApiError::not_found("entry not found"))?; - let artifact = detail - .artifacts - .get(artifact_index) - .ok_or(ApiError::not_found("artifact index out of range"))?; - let file_path = archive::resolve_artifact_path(&paths.store_path, artifact)?; - // ServeFile streams the file, handles Range requests (video seeking), - // sets Content-Type/ETag/Last-Modified. Error type is Infallible. - Ok(ServeFile::new(&file_path) - .oneshot(req) - .await - .unwrap() - .into_response()) -} - -#[derive(Debug, serde::Deserialize)] -struct CreateTagBody { - path: String, -} - -#[derive(Debug, serde::Deserialize)] -struct AssignTagBody { - tag_path: String, -} - -async fn list_tags( - State(state): State, - Path(archive_id): Path, -) -> Result>, ApiError> { - let mounted = mounted_archive(&state, &archive_id)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - Ok(Json(archive::list_tag_tree(&conn)?)) -} - -async fn create_tag_handler( - State(state): State, - Path(archive_id): Path, - Json(body): Json, -) -> Result<(StatusCode, Json), ApiError> { - if body.path.trim().is_empty() { - return Err(ApiError::bad_request("tag path must not be empty")); - } - let mounted = mounted_archive(&state, &archive_id)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - let tag = archive::create_tag(&conn, &body.path)?; - Ok((StatusCode::CREATED, Json(tag))) -} - -async fn list_entry_tags( - State(state): State, - Path((archive_id, entry_uid)): Path<(String, String)>, -) -> Result>, ApiError> { - let mounted = mounted_archive(&state, &archive_id)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - match archive::get_entry_tags(&conn, &entry_uid)? { - Some(tags) => Ok(Json(tags)), - None => Err(ApiError::not_found("entry not found")), - } -} - -async fn assign_entry_tag_handler( - State(state): State, - Path((archive_id, entry_uid)): Path<(String, String)>, - Json(body): Json, -) -> Result<(StatusCode, Json), ApiError> { - if body.tag_path.trim().is_empty() { - return Err(ApiError::bad_request("tag_path must not be empty")); - } - let mounted = mounted_archive(&state, &archive_id)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - match archive::assign_entry_tag(&conn, &entry_uid, &body.tag_path)? { - Some(tag) => Ok((StatusCode::CREATED, Json(tag))), - None => Err(ApiError::not_found("entry not found")), - } -} - -async fn remove_entry_tag_handler( - State(state): State, - Path((archive_id, entry_uid, tag_uid)): Path<(String, String, String)>, -) -> Result { - let mounted = mounted_archive(&state, &archive_id)?; - let conn = database::open_or_initialize(&mounted.archive_path)?; - if archive::remove_entry_tag(&conn, &entry_uid, &tag_uid)? { - Ok(StatusCode::NO_CONTENT) - } else { - Err(ApiError::not_found("entry or tag not found")) - } -} - -#[derive(Debug, serde::Deserialize)] -struct CaptureBody { - locator: String, -} - -async fn capture_handler( - State(state): State, - Path(archive_id): Path, - Json(body): Json, -) -> Result, ApiError> { - if body.locator.trim().is_empty() { - return Err(ApiError::bad_request("locator must not be empty")); - } - let mounted = mounted_archive(&state, &archive_id)?; - let archive_paths = archive::read_archive_paths(&mounted.archive_path) - .map_err(ApiError::from)?; - let result = capture::perform_capture(&archive_paths, &body.locator) - .map_err(ApiError::from)?; - Ok(Json(result)) -} - fn mounted_archive<'a>( state: &'a AppState, archive_id: &str, @@ -282,13 +101,6 @@ impl ApiError { message: message.to_string(), } } - - fn bad_request(message: &str) -> Self { - Self { - status: StatusCode::BAD_REQUEST, - message: message.to_string(), - } - } } impl From for ApiError @@ -325,7 +137,6 @@ mod tests { label: "Personal".to_string(), archive_path: std::path::PathBuf::from("/tmp/personal/.archivr"), }], - bind: None, }; let response = app(registry) .oneshot( @@ -354,627 +165,4 @@ mod tests { assert_eq!(response.status(), StatusCode::NOT_FOUND); } - - #[tokio::test] - async fn artifact_missing_archive_returns_404() { - let response = app(ServerRegistry::default()) - .oneshot( - Request::builder() - .uri("/api/archives/nope/entries/entry_abc/artifacts/0") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn artifact_missing_entry_returns_404() { - let dir = tempfile::tempdir().unwrap(); - archivr_core::archive::initialize_archive( - dir.path(), - &dir.path().join("store"), - "test", - false, - ) - .unwrap(); - let archive_path = dir.path().join(".archivr"); - let registry = ServerRegistry { - archives: vec![MountedArchive { - id: "test".to_string(), - label: "Test".to_string(), - archive_path, - }], - bind: None, - }; - let response = app(registry) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/entry_doesnotexist/artifacts/0") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn artifact_out_of_range_index_returns_404() { - let dir = tempfile::tempdir().unwrap(); - archivr_core::archive::initialize_archive( - dir.path(), - &dir.path().join("store"), - "test", - false, - ) - .unwrap(); - let archive_path = dir.path().join(".archivr"); - let registry = ServerRegistry { - archives: vec![MountedArchive { - id: "test".to_string(), - label: "Test".to_string(), - archive_path, - }], - bind: None, - }; - let response = app(registry) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/entry_doesnotexist/artifacts/99") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn artifact_serves_file_with_ok_status() { - // Initialize archive (creates .archivr dir, store dirs, and db) - let dir = tempfile::tempdir().unwrap(); - let store_path = dir.path().join("store"); - let paths = - archivr_core::archive::initialize_archive(dir.path(), &store_path, "test", false) - .unwrap(); - - // Write artifact file to the store - let artifact_relpath = "raw/a/b/test.html"; - let artifact_dir = store_path.join("raw").join("a").join("b"); - std::fs::create_dir_all(&artifact_dir).unwrap(); - std::fs::write(artifact_dir.join("test.html"), b"hello").unwrap(); - - // Populate the database with user, source identity, run, entry, blob, artifact - let conn = database::open_or_initialize(&paths.archive_path).unwrap(); - let user_id = database::ensure_default_user(&conn).unwrap(); - let source_identity_id = database::upsert_source_identity( - &conn, - "web", - "page", - Some("test-page"), - Some("https://example.com/page"), - "https://example.com/page", - ) - .unwrap(); - let run = database::create_archive_run(&conn, user_id, 1).unwrap(); - let entry = database::create_archived_entry( - &conn, - &database::NewEntry { - source_identity_id, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: "web".to_string(), - entity_kind: "page".to_string(), - title: Some("Test Page".to_string()), - visibility: "private".to_string(), - representation_kind: "html".to_string(), - source_metadata_json: r#"{"source":"test"}"#.to_string(), - display_metadata_json: None, - }, - ) - .unwrap(); - let blob_id = database::upsert_blob( - &conn, - &database::BlobRecord { - sha256: "abc123testblob".to_string(), - byte_size: 18, - mime_type: Some("text/html".to_string()), - extension: Some("html".to_string()), - raw_relpath: artifact_relpath.to_string(), - }, - ) - .unwrap(); - database::add_entry_artifact( - &conn, - &database::NewArtifact { - entry_id: entry.id, - artifact_role: "primary_media".to_string(), - storage_area: "raw".to_string(), - relpath: artifact_relpath.to_string(), - blob_id: Some(blob_id), - logical_path: None, - metadata_json: None, - }, - ) - .unwrap(); - drop(conn); // release before the HTTP handler opens the same db file - - let registry = ServerRegistry { - archives: vec![MountedArchive { - id: "test".to_string(), - label: "Test".to_string(), - archive_path: paths.archive_path.clone(), - }], - bind: None, - }; - let uri = format!( - "/api/archives/test/entries/{}/artifacts/0", - entry.entry_uid - ); - let response = app(registry) - .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - } - - #[tokio::test] - async fn search_missing_archive_returns_404() { - let response = app(ServerRegistry::default()) - .oneshot( - Request::builder() - .uri("/api/archives/nope/entries/search?q=anything") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn search_empty_q_returns_ok() { - let dir = tempfile::tempdir().unwrap(); - archivr_core::archive::initialize_archive( - dir.path(), - &dir.path().join("store"), - "test", - false, - ) - .unwrap(); - let archive_path = dir.path().join(".archivr"); - let registry = ServerRegistry { - archives: vec![MountedArchive { - id: "test".to_string(), - label: "Test".to_string(), - archive_path, - }], - bind: None, - }; - let response = app(registry) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/search") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - } - - #[tokio::test] - async fn search_unknown_prefix_returns_400() { - let dir = tempfile::tempdir().unwrap(); - archivr_core::archive::initialize_archive( - dir.path(), - &dir.path().join("store"), - "test", - false, - ) - .unwrap(); - let archive_path = dir.path().join(".archivr"); - let registry = ServerRegistry { - archives: vec![MountedArchive { - id: "test".to_string(), - label: "Test".to_string(), - archive_path, - }], - bind: None, - }; - let response = app(registry) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/search?q=unknownprefix%3Aval") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - // ---- helpers ---- - - fn make_test_registry(dir: &tempfile::TempDir) -> (ServerRegistry, std::path::PathBuf) { - let paths = archivr_core::archive::initialize_archive( - dir.path(), - &dir.path().join("store"), - "test", - false, - ) - .unwrap(); - let registry = ServerRegistry { - archives: vec![MountedArchive { - id: "test".to_string(), - label: "Test".to_string(), - archive_path: paths.archive_path.clone(), - }], - bind: None, - }; - (registry, paths.archive_path) - } - - fn make_test_entry(archive_path: &std::path::Path) -> archivr_core::database::ArchivedEntry { - let conn = database::open_or_initialize(archive_path).unwrap(); - let user_id = database::ensure_default_user(&conn).unwrap(); - let run = database::create_archive_run(&conn, user_id, 1).unwrap(); - let si = database::upsert_source_identity( - &conn, "web", "page", None, - Some("https://example.com/test"), - "https://example.com/test", - ) - .unwrap(); - database::create_archived_entry( - &conn, - &database::NewEntry { - source_identity_id: si, - archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, - created_by_user_id: user_id, - owned_by_user_id: user_id, - source_kind: "web".to_string(), - entity_kind: "page".to_string(), - title: Some("Test Entry".to_string()), - visibility: "private".to_string(), - representation_kind: "html".to_string(), - source_metadata_json: "{}".to_string(), - display_metadata_json: None, - }, - ) - .unwrap() - } - - async fn body_json(response: axum::response::Response) -> serde_json::Value { - let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .unwrap(); - serde_json::from_slice(&bytes).unwrap() - } - - fn json_body(payload: &serde_json::Value) -> Body { - Body::from(serde_json::to_vec(payload).unwrap()) - } - - // ---- tag route tests ---- - - #[tokio::test] - async fn test_list_tags_unknown_archive() { - let response = app(ServerRegistry::default()) - .oneshot( - Request::builder() - .uri("/api/archives/ghost/tags") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn test_create_tag_unknown_archive() { - let response = app(ServerRegistry::default()) - .oneshot( - Request::builder() - .method("POST") - .uri("/api/archives/ghost/tags") - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"path": "/science"}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn test_create_tag_empty_path() { - let dir = tempfile::tempdir().unwrap(); - let (registry, _) = make_test_registry(&dir); - let response = app(registry) - .oneshot( - Request::builder() - .method("POST") - .uri("/api/archives/test/tags") - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"path": ""}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - #[tokio::test] - async fn test_tag_round_trip() { - let dir = tempfile::tempdir().unwrap(); - let (registry, _) = make_test_registry(&dir); - - let create_response = app(registry.clone()) - .oneshot( - Request::builder() - .method("POST") - .uri("/api/archives/test/tags") - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"path": "/science"}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(create_response.status(), StatusCode::CREATED); - - let list_response = app(registry.clone()) - .oneshot( - Request::builder() - .uri("/api/archives/test/tags") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(list_response.status(), StatusCode::OK); - let tree = body_json(list_response).await; - let slugs: Vec<&str> = tree - .as_array() - .unwrap() - .iter() - .map(|n| n["tag"]["slug"].as_str().unwrap()) - .collect(); - assert!(slugs.contains(&"science"), "expected 'science' in tag tree, got {slugs:?}"); - } - - #[tokio::test] - async fn test_entry_tag_assign_and_remove() { - let dir = tempfile::tempdir().unwrap(); - let (registry, archive_path) = make_test_registry(&dir); - let entry = make_test_entry(&archive_path); - let entry_uid = entry.entry_uid.clone(); - let entry_tags_uri = format!("/api/archives/test/entries/{entry_uid}/tags"); - - // Assign tag - let assign_response = app(registry.clone()) - .oneshot( - Request::builder() - .method("POST") - .uri(&entry_tags_uri) - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"tag_path": "/science"}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(assign_response.status(), StatusCode::CREATED); - let assigned_tag = body_json(assign_response).await; - let tag_uid = assigned_tag["tag_uid"].as_str().unwrap().to_string(); - - // List entry tags — should contain the assigned tag - let list_response = app(registry.clone()) - .oneshot( - Request::builder() - .uri(&entry_tags_uri) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(list_response.status(), StatusCode::OK); - let tags = body_json(list_response).await; - assert_eq!(tags.as_array().unwrap().len(), 1); - - // Remove tag - let delete_uri = format!("{entry_tags_uri}/{tag_uid}"); - let delete_response = app(registry.clone()) - .oneshot( - Request::builder() - .method("DELETE") - .uri(&delete_uri) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(delete_response.status(), StatusCode::NO_CONTENT); - - // List entry tags again — should be empty - let list2_response = app(registry.clone()) - .oneshot( - Request::builder() - .uri(&entry_tags_uri) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(list2_response.status(), StatusCode::OK); - let tags2 = body_json(list2_response).await; - assert!(tags2.as_array().unwrap().is_empty(), "tags should be empty after removal"); - } - - #[tokio::test] - async fn test_search_with_tag_param() { - let dir = tempfile::tempdir().unwrap(); - let (registry, archive_path) = make_test_registry(&dir); - let entry = make_test_entry(&archive_path); - let entry_uid = entry.entry_uid.clone(); - - // Assign /science tag to entry - let assign_resp = app(registry.clone()) - .oneshot( - Request::builder() - .method("POST") - .uri(format!("/api/archives/test/entries/{entry_uid}/tags")) - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"tag_path": "/science"}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(assign_resp.status(), StatusCode::CREATED, "assign tag should return 201"); - - // Search with ?tag=/science — entry should appear - let response = app(registry.clone()) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/search?tag=%2Fscience") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let results = body_json(response).await; - assert_eq!( - results.as_array().unwrap().len(), - 1, - "expected 1 result for /science tag, got {}", - results.as_array().unwrap().len() - ); - - // Search with ?tag=/art — should return empty - let response2 = app(registry.clone()) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/search?tag=%2Fart") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response2.status(), StatusCode::OK); - let results2 = body_json(response2).await; - assert!( - results2.as_array().unwrap().is_empty(), - "expected 0 results for /art tag" - ); - } - - #[tokio::test] - async fn test_list_entry_tags_unknown_entry() { - let dir = tempfile::tempdir().unwrap(); - let (registry, _) = make_test_registry(&dir); - let response = app(registry) - .oneshot( - Request::builder() - .uri("/api/archives/test/entries/ghost_uid/tags") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn test_assign_entry_tag_unknown_entry() { - let dir = tempfile::tempdir().unwrap(); - let (registry, _) = make_test_registry(&dir); - let response = app(registry) - .oneshot( - Request::builder() - .method("POST") - .uri("/api/archives/test/entries/ghost_uid/tags") - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"tag_path": "/science"}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn test_assign_entry_tag_empty_tag_path() { - let dir = tempfile::tempdir().unwrap(); - let (registry, archive_path) = make_test_registry(&dir); - let entry = make_test_entry(&archive_path); - let entry_uid = entry.entry_uid.clone(); - let response = app(registry) - .oneshot( - Request::builder() - .method("POST") - .uri(format!("/api/archives/test/entries/{entry_uid}/tags")) - .header("content-type", "application/json") - .body(json_body(&serde_json::json!({"tag_path": ""}))) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - #[tokio::test] - async fn test_remove_entry_tag_unknown_entry() { - let dir = tempfile::tempdir().unwrap(); - let (registry, _) = make_test_registry(&dir); - let response = app(registry) - .oneshot( - Request::builder() - .method("DELETE") - .uri("/api/archives/test/entries/ghost_uid/tags/ghost_tag_uid") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn capture_rejects_empty_locator() { - let response = app(ServerRegistry::default()) - .oneshot( - Request::builder() - .method("POST") - .uri("/api/archives/test/captures") - .header("content-type", "application/json") - .body(Body::from(r#"{"locator":""}"#)) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - #[tokio::test] - async fn capture_rejects_unknown_archive() { - let response = app(ServerRegistry::default()) - .oneshot( - Request::builder() - .method("POST") - .uri("/api/archives/nonexistent/captures") - .header("content-type", "application/json") - .body(Body::from(r#"{"locator":"tweet:1234567890"}"#)) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } } diff --git a/crates/archivr-server/static/app.js b/crates/archivr-server/static/app.js index 1934341..68b1616 100644 --- a/crates/archivr-server/static/app.js +++ b/crates/archivr-server/static/app.js @@ -2,11 +2,9 @@ const state = { archives: [], archiveId: null, entries: [], + filteredEntries: [], selectedEntryUid: null, - selectedEntry: null, - tagFilter: null, }; -let selectSeq = 0; const archiveSwitcher = document.querySelector("#archive-switcher"); const entriesBody = document.querySelector("#entries-body"); @@ -16,17 +14,6 @@ const navButtons = document.querySelectorAll(".nav-link"); const searchInput = document.querySelector("#search"); const resultCount = document.querySelector("#result-count"); const adminArchives = document.querySelector("#admin-archives"); -const tagTree = document.querySelector("#tag-tree"); -const entryTagsEl = document.querySelector("#entry-tags"); -const assignTagForm = document.querySelector("#assign-tag-form"); -const assignTagInput = document.querySelector("#assign-tag-input"); -const assignTagBtn = document.querySelector("#assign-tag-btn"); -const captureButton = document.querySelector('.capture-button'); -const captureDialog = document.querySelector('#capture-dialog'); -const captureLocatorInput = document.querySelector('#capture-locator'); -const captureSubmitBtn = document.querySelector('#capture-submit-btn'); -const captureCancelBtn = document.querySelector('#capture-cancel-btn'); -const captureError = document.querySelector('#capture-error'); function formatBytes(bytes) { if (!bytes) return "0 B"; @@ -84,26 +71,33 @@ function renderArchives() { } } +function applyEntryFilter() { + const query = searchInput.value.trim().toLowerCase(); + if (!query) { + state.filteredEntries = state.entries; + return; + } + state.filteredEntries = state.entries.filter((entry) => { + const haystack = [ + entry.title, + entry.original_url, + entry.entry_uid, + entry.source_kind, + entry.entity_kind, + entry.visibility, + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return haystack.includes(query); + }); +} function renderEntries() { entriesBody.innerHTML = ""; - if (state.entries.length === 0 && searchInput.value.trim()) { - resultCount.textContent = "No results."; - } else { - resultCount.textContent = `${state.entries.length} entries`; - } - if (state.tagFilter) { - const badge = document.createElement("button"); - badge.className = "tag-filter-badge"; - badge.textContent = `× ${state.tagFilter}`; - badge.addEventListener("click", () => { - state.tagFilter = null; - if (state.archiveId) loadEntries(searchInput.value); - }); - resultCount.appendChild(badge); - } + resultCount.textContent = `${state.filteredEntries.length} entries`; - for (const entry of state.entries) { + for (const entry of state.filteredEntries) { const row = document.createElement("tr"); row.tabIndex = 0; row.dataset.entryUid = entry.entry_uid; @@ -138,139 +132,30 @@ function renderEntries() { function renderContextDetail(detail) { contextBody.innerHTML = ""; + const title = document.createElement("strong"); + title.textContent = valueText(detail.summary.title) || valueText(detail.summary.entry_uid); + contextBody.append(title); - // Title - const titleEl = document.createElement("strong"); - titleEl.className = "rail-entry-title"; - titleEl.textContent = - valueText(detail.summary.title) || valueText(detail.summary.entry_uid); - contextBody.append(titleEl); - - // Metadata section - const metaSection = document.createElement("div"); - metaSection.className = "rail-section"; - - if (detail.summary.original_url) { - const urlRow = document.createElement("div"); - urlRow.className = "rail-item"; - const urlLabel = document.createElement("span"); - urlLabel.className = "rail-label"; - urlLabel.textContent = "Original URL"; - const urlLink = document.createElement("a"); - urlLink.href = detail.summary.original_url; - urlLink.target = "_blank"; - urlLink.rel = "noopener noreferrer"; - urlLink.className = "rail-url-link"; - urlLink.textContent = detail.summary.original_url; - urlRow.append(urlLabel, document.createTextNode(": "), urlLink); - metaSection.append(urlRow); - } - - const metaFields = [ - ["Added", detail.summary.archived_at], - ["Source", detail.summary.source_kind], + const items = [ ["Type", detail.summary.entity_kind], ["Visibility", detail.summary.visibility], + ["Artifacts", detail.artifacts.length], ["Structured root", detail.structured_root_relpath], ]; - for (const [label, value] of metaFields) { + + for (const [label, value] of items) { const item = document.createElement("div"); item.className = "rail-item"; - const labelEl = document.createElement("span"); - labelEl.className = "rail-label"; - labelEl.textContent = label; - item.append(labelEl, document.createTextNode(`: ${valueText(value)}`)); - metaSection.append(item); - } - contextBody.append(metaSection); - - // Artifacts section - if (detail.artifacts.length > 0) { - const artifactsSection = document.createElement("div"); - artifactsSection.className = "rail-section"; - const artifactsHeading = document.createElement("div"); - artifactsHeading.className = "rail-section-heading"; - artifactsHeading.textContent = `Artifacts (${detail.artifacts.length})`; - artifactsSection.append(artifactsHeading); - const list = document.createElement("ul"); - list.className = "artifact-list"; - detail.artifacts.forEach((artifact, index) => { - const li = document.createElement("li"); - const a = document.createElement("a"); - a.href = `/api/archives/${state.archiveId}/entries/${detail.summary.entry_uid}/artifacts/${index}`; - a.target = "_blank"; - a.rel = "noopener noreferrer"; - a.className = "artifact-link"; - const roleName = artifact.artifact_role.replace(/_/g, " "); - const size = - artifact.byte_size != null ? ` (${formatBytes(artifact.byte_size)})` : ""; - a.textContent = `${roleName}${size}`; - li.append(a); - list.append(li); - }); - artifactsSection.append(list); - contextBody.append(artifactsSection); - } else { - const noArtifacts = document.createElement("div"); - noArtifacts.className = "rail-item muted"; - noArtifacts.textContent = "No artifacts."; - contextBody.append(noArtifacts); - } -} - -function renderEntryTags(tags, entryUid) { - entryTagsEl.innerHTML = ""; - if (!tags.length) { - entryTagsEl.textContent = "No tags."; - return; - } - for (const tag of tags) { - const pill = document.createElement("span"); - pill.className = "tag-pill"; - pill.textContent = tag.name; - pill.title = tag.full_path; - const removeBtn = document.createElement("button"); - removeBtn.className = "remove-tag"; - removeBtn.textContent = "×"; - removeBtn.title = `Remove tag ${tag.full_path}`; - removeBtn.addEventListener("click", async () => { - const resp = await fetch( - `/api/archives/${state.archiveId}/entries/${entryUid}/tags/${tag.tag_uid}`, - { method: "DELETE" } - ); - if (!resp.ok) { - removeBtn.title = `Remove failed (${resp.status})`; - return; - } - const updated = await getJson( - `/api/archives/${state.archiveId}/entries/${entryUid}/tags` - ); - renderEntryTags(updated, entryUid); - loadTagTree(); - }); - pill.appendChild(removeBtn); - entryTagsEl.appendChild(pill); + item.textContent = `${label}: ${valueText(value)}`; + contextBody.append(item); } } async function selectEntry(entry) { - const seq = ++selectSeq; state.selectedEntryUid = entry.entry_uid; - state.selectedEntry = entry; renderEntries(); - const detail = await getJson( - `/api/archives/${state.archiveId}/entries/${entry.entry_uid}` - ); - if (seq !== selectSeq) return; + const detail = await getJson(`/api/archives/${state.archiveId}/entries/${entry.entry_uid}`); renderContextDetail(detail); - entryTagsEl.hidden = false; - assignTagForm.hidden = false; - entryTagsEl.innerHTML = ""; - const tags = await getJson( - `/api/archives/${state.archiveId}/entries/${entry.entry_uid}/tags` - ); - if (seq !== selectSeq) return; - renderEntryTags(tags, entry.entry_uid); } async function loadRuns() { @@ -287,68 +172,12 @@ async function loadRuns() { } } -async function loadEntries(q = "") { - const trimmed = q.trim(); - const params = new URLSearchParams(); - if (trimmed) params.set("q", trimmed); - if (state.tagFilter) params.set("tag", state.tagFilter); - const url = - trimmed || state.tagFilter - ? `/api/archives/${state.archiveId}/entries/search?${params}` - : `/api/archives/${state.archiveId}/entries`; - searchInput.setAttribute("aria-busy", "true"); - try { - state.entries = await getJson(url); - } catch (err) { - resultCount.textContent = "Search failed. Try again."; - state.entries = []; - } finally { - searchInput.removeAttribute("aria-busy"); - } +async function loadEntries() { + state.entries = await getJson(`/api/archives/${state.archiveId}/entries`); + state.selectedEntryUid = null; + applyEntryFilter(); renderEntries(); -} - -async function loadTagTree() { - if (!state.archiveId) return; - const nodes = await getJson(`/api/archives/${state.archiveId}/tags`); - tagTree.innerHTML = ""; - renderTagTree(nodes, tagTree); -} - -function renderTagTree(nodes, container) { - if (!nodes.length) { - container.textContent = "No tags yet."; - return; - } - const ul = document.createElement("ul"); - ul.className = "tag-tree-list"; - for (const node of nodes) { - const li = document.createElement("li"); - const btn = document.createElement("button"); - btn.className = "tag-node-btn"; - if (state.tagFilter === node.tag.full_path) btn.classList.add("is-active"); - btn.textContent = node.tag.name; - btn.title = node.tag.full_path; - btn.addEventListener("click", () => { - if (state.tagFilter === node.tag.full_path) { - state.tagFilter = null; - } else { - state.tagFilter = node.tag.full_path; - } - // Switch to archive view and reload - switchView("archive"); - if (state.archiveId) loadEntries(searchInput.value); - }); - li.appendChild(btn); - if (node.children?.length) { - const childContainer = document.createElement("div"); - childContainer.className = "tag-children"; - renderTagTree(node.children, childContainer); - li.appendChild(childContainer); - } - ul.appendChild(li); - } - container.appendChild(ul); + contextBody.textContent = "Select an entry."; } async function loadArchives() { @@ -358,118 +187,32 @@ async function loadArchives() { if (state.archiveId) { await loadEntries(); await loadRuns(); - loadTagTree(); } else { contextBody.textContent = "No archives mounted."; resultCount.textContent = "0 entries"; } } -function debounce(fn, ms) { - let timer; - return (...args) => { - clearTimeout(timer); - timer = setTimeout(() => fn(...args), ms); - }; -} - archiveSwitcher.addEventListener("change", async () => { - state.tagFilter = null; - state.selectedEntry = null; - state.selectedEntryUid = null; - entryTagsEl.hidden = true; - assignTagForm.hidden = true; state.archiveId = archiveSwitcher.value; await loadEntries(); await loadRuns(); - loadTagTree(); }); -const debouncedSearch = debounce((q) => { - if (state.archiveId) loadEntries(q); -}, 300); - searchInput.addEventListener("input", () => { - debouncedSearch(searchInput.value); + applyEntryFilter(); + renderEntries(); }); -function switchView(name) { - navButtons.forEach(b => b.classList.toggle("is-active", b.dataset.view === name)); - document.querySelectorAll(".view").forEach(v => v.classList.remove("is-active")); - document.querySelector(`#${name}-view`)?.classList.add("is-active"); -} - navButtons.forEach((button) => { button.addEventListener("click", () => { - switchView(button.dataset.view); - if (button.dataset.view === "tags") loadTagTree(); + navButtons.forEach((candidate) => candidate.classList.remove("is-active")); + document.querySelectorAll(".view").forEach((view) => view.classList.remove("is-active")); + button.classList.add("is-active"); + document.querySelector(`#${button.dataset.view}-view`).classList.add("is-active"); }); }); -assignTagBtn.addEventListener("click", async () => { - const path = assignTagInput.value.trim(); - if (!path || !state.selectedEntry) return; - const resp = await fetch( - `/api/archives/${state.archiveId}/entries/${state.selectedEntry.entry_uid}/tags`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ tag_path: path }), - } - ); - if (resp.ok) { - assignTagInput.setCustomValidity(""); - assignTagInput.value = ""; - const tags = await getJson( - `/api/archives/${state.archiveId}/entries/${state.selectedEntry.entry_uid}/tags` - ); - renderEntryTags(tags, state.selectedEntry.entry_uid); - loadTagTree(); - } else { - assignTagInput.setCustomValidity(`Failed to add tag (${resp.status})`); - assignTagInput.reportValidity(); - } -}); - -captureButton.addEventListener('click', () => { - captureLocatorInput.value = ''; - captureError.hidden = true; - captureDialog.showModal(); -}); - -captureCancelBtn.addEventListener('click', () => captureDialog.close()); - -captureSubmitBtn.addEventListener('click', async () => { - const locator = captureLocatorInput.value.trim(); - if (!locator) { - captureError.textContent = 'Enter a locator.'; - captureError.hidden = false; - return; - } - captureSubmitBtn.disabled = true; - captureSubmitBtn.textContent = 'Capturing\u2026'; - captureError.hidden = true; - try { - const res = await fetch(`/api/archives/${state.archiveId}/captures`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ locator }), - }); - if (!res.ok) { - const msg = await res.text(); - throw new Error(msg || `HTTP ${res.status}`); - } - captureDialog.close(); - await Promise.all([loadEntries(searchInput.value), loadRuns()]); - } catch (e) { - captureError.textContent = e.message; - captureError.hidden = false; - } finally { - captureSubmitBtn.disabled = false; - captureSubmitBtn.textContent = 'Capture'; - } -}); - loadArchives().catch((error) => { contextBody.textContent = `Failed to load archives: ${error.message}`; }); diff --git a/crates/archivr-server/static/index.html b/crates/archivr-server/static/index.html index 9aa2777..021ef15 100644 --- a/crates/archivr-server/static/index.html +++ b/crates/archivr-server/static/index.html @@ -14,7 +14,6 @@ - @@ -67,37 +66,14 @@

Mounted Archives

- -
-
-
- -
-

Capture

- - - -
- - -
-
- - diff --git a/crates/archivr-server/static/styles.css b/crates/archivr-server/static/styles.css index 46f246f..c0e9216 100644 --- a/crates/archivr-server/static/styles.css +++ b/crates/archivr-server/static/styles.css @@ -312,277 +312,3 @@ select { min-width: 860px; } } - -.rail-entry-title { - display: block; - font-size: 15px; - font-weight: 700; - color: var(--ink); - margin-bottom: 12px; - line-height: 1.4; -} - -.rail-section { - margin-bottom: 18px; -} - -.rail-section-heading { - font-size: 11px; - font-weight: 800; - text-transform: uppercase; - letter-spacing: 0.04em; - color: var(--accent); - margin-bottom: 6px; -} - -.rail-label { - font-weight: 600; - color: var(--ink); -} - -.rail-url-link { - color: var(--accent); - word-break: break-all; - font-size: 13px; -} - -.artifact-list { - list-style: none; - margin: 0; - padding: 0; - display: flex; - flex-direction: column; - gap: 4px; -} - -.artifact-link { - display: block; - padding: 6px 8px; - background: var(--paper-3); - border: 1px solid var(--line); - color: var(--accent); - text-decoration: none; - font-size: 13px; - border-radius: 3px; -} - -.artifact-link:hover { - background: var(--line); - text-decoration: underline; -} -/* Search loading state */ -.search-input[aria-busy="true"] { - cursor: progress; -} - -/* Tag tree */ -.tag-tree { - padding: 12px; -} - -.tag-tree-list { - list-style: none; - margin: 0; - padding: 0; -} - -.tag-children { - padding-left: 16px; -} - -.tag-node-btn { - border: 0; - background: transparent; - color: var(--link); - cursor: pointer; - padding: 3px 0; - text-align: left; - font-size: 13px; - display: block; - width: 100%; -} - -.tag-node-btn:hover { - text-decoration: underline; -} - -.tag-node-btn.is-active { - font-weight: 600; - color: var(--ink); -} - -/* Tag pills in context rail */ -.entry-tags { - display: flex; - flex-wrap: wrap; - gap: 6px; - padding: 8px 0; -} - -.tag-pill { - display: inline-flex; - align-items: center; - gap: 4px; - background: var(--paper-2); - border: 1px solid var(--line); - border-radius: 3px; - font-size: 12px; - padding: 2px 6px; - color: var(--ink); -} - -.remove-tag { - border: 0; - background: none; - cursor: pointer; - color: var(--muted); - padding: 0; - font-size: 14px; - line-height: 1; -} - -.remove-tag:hover { - color: var(--accent); -} - -/* Assign tag form */ -.assign-tag-form { - display: flex; - gap: 6px; - padding: 6px 0 8px; - border-top: 1px solid var(--line-soft); - margin-top: 4px; -} - -.assign-tag-input { - flex: 1; - min-width: 0; - border: 1px solid var(--line); - background: var(--paper-3); - color: var(--ink); - padding: 4px 7px; - font-size: 12px; -} - -.assign-tag-btn { - border: 1px solid var(--line); - background: var(--paper-2); - color: var(--ink); - cursor: pointer; - padding: 4px 10px; - font-size: 12px; - white-space: nowrap; -} - -.assign-tag-btn:hover { - background: var(--paper); -} - -/* Tag filter badge in search row */ -.tag-filter-badge { - display: inline-flex; - align-items: center; - gap: 4px; - background: var(--accent); - color: #fff; - border: 0; - border-radius: 3px; - font-size: 12px; - padding: 2px 8px; - cursor: pointer; - margin-left: 8px; -} - -.tag-filter-badge:hover { - opacity: 0.85; -} - -/* Capture dialog */ -.capture-dialog { - border: 2px solid var(--ink); - background: var(--paper); - padding: 0; - min-width: 360px; - max-width: 520px; - box-shadow: 4px 4px 0 var(--ink); -} - -.capture-dialog::backdrop { - background: rgba(0, 0, 0, 0.45); -} - -.capture-dialog-inner { - padding: 28px; -} - -.capture-dialog-title { - margin: 0 0 16px; - font-size: 18px; - font-family: Georgia, "Times New Roman", serif; -} - -.capture-label { - display: block; - font-size: 13px; - font-weight: 600; - margin-bottom: 6px; - color: var(--muted); - text-transform: uppercase; - letter-spacing: 0.04em; -} - -.capture-input { - width: 100%; - height: 42px; - border: 2px solid var(--ink); - background: var(--paper-3); - color: var(--ink); - padding: 0 12px; - font-size: 15px; - margin-bottom: 10px; - outline-offset: 3px; -} - -.capture-error { - color: var(--accent); - font-size: 13px; - margin-bottom: 10px; - min-height: 18px; -} - -.capture-actions { - display: flex; - gap: 10px; - justify-content: flex-end; - margin-top: 16px; -} - -.capture-cancel { - border: 1px solid var(--line); - background: none; - color: var(--ink); - padding: 8px 18px; - cursor: pointer; -} - -.capture-cancel:hover { - background: var(--paper-2); -} - -.capture-submit { - border: 0; - background: var(--ink); - color: var(--paper); - padding: 8px 20px; - cursor: pointer; - font-weight: 600; -} - -.capture-submit:hover { - opacity: 0.85; -} - -.capture-submit:disabled { - opacity: 0.45; - cursor: default; -} diff --git a/docs/README.md b/docs/README.md index 7917645..c6d1eba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,79 +56,6 @@ This project aims to provide a reliable solution for archiving important data fr - Direct platform URLs - Platform shorthand inputs such as `tweet:...`, `yt:...`, or `instagram:...` -## Running Archivr - -Archivr currently ships as two binaries: - -- `archivr` - - The CLI for creating and writing to one archive. - - Use this for `init` and `archive`. -- `archivr-server` - - The web server for reading one or more existing archives through the browser UI. - - Use this after archives already exist. - -With Nix, run the CLI with: - -```sh -nix run .#archivr -- init ./my-archive --name "My Archive" -nix run .#archivr -- archive file:///absolute/path/to/file.pdf -``` - -Run the web server with: - -```sh -nix run .#archivr-server -- ./archivr-server.toml -``` - -The server expects a TOML registry file. If no path is passed, it reads `./archivr-server.toml`. - -Example: - -```toml -[[archives]] -id = "personal" -label = "Personal" -archive_path = "/absolute/path/to/my-archive/.archivr" -``` - -Then open: - -```text -http://127.0.0.1:8080 -``` - -When installed through Nix, `archivr-server` is wrapped so it can find the static web UI assets automatically. The wrapper sets `ARCHIVR_STATIC_DIR` to the installed static asset directory. Running from source with `cargo run -p archivr-server` falls back to `crates/archivr-server/static`. - -### Security and Deployment - -`archivr-server` is a **local-only tool by default**. It binds to `127.0.0.1:8080` and has no authentication or access control. Do not expose it to a public network or a shared LAN without understanding the risks. - -**Changing the bind address** - -You can set the bind address in your TOML config: - -```toml -# Optional. Default: 127.0.0.1:8080 -# Only change this if you know what you are doing — the server has no authentication. -bind = "127.0.0.1:9090" -``` - -Or override it with the `ARCHIVR_BIND` environment variable: - -```sh -ARCHIVR_BIND=127.0.0.1:9090 nix run .#archivr-server -- ./archivr-server.toml -``` - -If the server is started with a non-loopback address (e.g. `0.0.0.0`), it prints a warning to stderr: - -```text -warn: archivr-server is bound to 0.0.0.0:8080 — this server has no authentication. Only expose it on a trusted network. -``` - -**When will auth be added?** - -Auth and session handling will be designed when remote or public hosting becomes a real requirement. Until then, keep the server on loopback. See `crates/archivr-server/src/routes.rs` for the route classification that will guide where middleware is applied. - ### Supported Platforms - Local files: `file:///absolute/path/to/file.ext` diff --git a/flake.nix b/flake.nix index 368dd8c..ca122fb 100644 --- a/flake.nix +++ b/flake.nix @@ -129,31 +129,18 @@ pname = "archivr-server-wrapped"; inherit version; nativeBuildInputs = [ pkgs.makeWrapper ]; - buildInputs = [ tweetPython ]; phases = [ "installPhase" ]; installPhase = '' mkdir -p $out/bin $out/libexec/archivr-server $out/share/archivr-server/static cp ${archivr_server_unwrapped}/bin/archivr-server $out/libexec/archivr-server/archivr-server - cp ${./vendor/twitter/scrape_user_tweet_contents.py} $out/libexec/archivr-server/scrape_user_tweet_contents.py - chmod +x $out/libexec/archivr-server/scrape_user_tweet_contents.py cp -r ${./crates/archivr-server/static}/* $out/share/archivr-server/static/ makeWrapper $out/libexec/archivr-server/archivr-server $out/bin/archivr-server \ - --set ARCHIVR_STATIC_DIR $out/share/archivr-server/static \ - --set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \ - --set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr-server/scrape_user_tweet_contents.py + --set ARCHIVR_STATIC_DIR $out/share/archivr-server/static ''; }; - archivr-all = pkgs.symlinkJoin { - name = "archivr-all"; - paths = [ - archivr - archivr_server - ]; - }; in { - default = archivr-all; - archivr-all = archivr-all; + default = archivr; archivr = archivr; archivr-cli = archivr; archivr-cli-unwrapped = archivr_cli_unwrapped;