From 3e289383a9cf9b4f7ed2d539e2cc17ca43a1893f Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:49:34 +0200 Subject: [PATCH] feat: platform-aware entry titling for all 13 source types - Add PlatformMetadata struct and generate_entry_title() covering all 13 Source variants (YouTubeVideo, YouTubePlaylist, YouTubeChannel, X, Tweet, TweetThread, Instagram, Facebook, TikTok, Reddit, Snapchat, Local, Other) - Add downloader/metadata.rs: extract_from_ytdlp_json() parses yt-dlp --dump-json output into PlatformMetadata; extracts Reddit subreddit from webpage_url via regex - Add ytdlp::fetch_metadata(): separate --dump-json invocation that does not interfere with the actual download call - Extend record_media_entry() with title: Option param; wire yt-dlp metadata fetch + local filename extraction in perform_capture - Restructure record_tweet_entry() to read tweet JSON before entry creation; extract title via tweet_metadata_from_json() - 16 title-generation unit tests + 6 metadata extraction unit tests --- crates/archivr-core/src/capture.rs | 265 +++++++++++++++++- .../archivr-core/src/downloader/metadata.rs | 110 ++++++++ crates/archivr-core/src/downloader/mod.rs | 1 + crates/archivr-core/src/downloader/ytdlp.rs | 22 ++ 4 files changed, 394 insertions(+), 4 deletions(-) create mode 100644 crates/archivr-core/src/downloader/metadata.rs diff --git a/crates/archivr-core/src/capture.rs b/crates/archivr-core/src/capture.rs index a1690f7..e345a2a 100644 --- a/crates/archivr-core/src/capture.rs +++ b/crates/archivr-core/src/capture.rs @@ -31,6 +31,65 @@ pub struct CaptureResult { pub status: String, } +#[derive(Debug, Clone, Default)] +pub struct PlatformMetadata { + /// Uploader / creator handle (without @) + pub author: Option, + /// Video title, playlist name, post title, or filename + pub title: Option, + /// Tweet text, Instagram caption, TikTok caption + pub caption: Option, + /// Reddit subreddit name (without r/) + pub subreddit: Option, + /// Reddit post author handle (without u/) + pub post_author: Option, +} + +impl PlatformMetadata { + /// Returns caption trimmed to 100 chars followed by "..." when longer. + pub fn caption_excerpt(&self) -> Option { + self.caption.as_ref().and_then(|c| { + let t = c.trim(); + if t.is_empty() { + None + } else if t.len() > 100 { + Some(format!("{}...", &t[..100])) + } else { + Some(t.to_string()) + } + }) + } +} + +fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String { + match source { + Source::YouTubeVideo => meta.title.clone().unwrap_or_else(|| "YouTube Video".to_string()), + Source::YouTubePlaylist => meta.title.clone().unwrap_or_else(|| "YouTube Playlist".to_string()), + Source::YouTubeChannel => format!( + "Archival of {}", + meta.author.as_deref().unwrap_or("Unknown Channel") + ), + Source::X => format!("X Media by {}", meta.author.as_deref().unwrap_or("unknown")), + Source::Tweet => { + let excerpt = meta.caption_excerpt().unwrap_or_else(|| "Tweet".to_string()); + format!("{} \u{2014} @{}", excerpt, meta.author.as_deref().unwrap_or("unknown")) + } + Source::TweetThread => format!("Thread by @{}", meta.author.as_deref().unwrap_or("unknown")), + Source::Instagram => format!("Post by @{}", meta.author.as_deref().unwrap_or("unknown")), + Source::Facebook => format!("Post by {}", meta.author.as_deref().unwrap_or("unknown")), + Source::TikTok => format!("TikTok by @{}", meta.author.as_deref().unwrap_or("unknown")), + Source::Reddit => format!( + "{} \u{2014} r/{} (u/{})", + meta.title.as_deref().unwrap_or("Reddit Post"), + meta.subreddit.as_deref().unwrap_or("reddit"), + meta.post_author.as_deref().unwrap_or("unknown") + ), + Source::Snapchat => format!("Snap by {}", meta.author.as_deref().unwrap_or("unknown")), + Source::Local => meta.title.clone().unwrap_or_else(|| "Local File".to_string()), + Source::Other => "Archived Content".to_string(), + } +} + fn expand_shorthand_to_url(path: &str, source: &Source) -> String { // YouTube shorthands: yt:video/ID, yt:playlist/ID, yt:@handle, yt:channel/ID, etc. if matches!(source, Source::YouTubeVideo | Source::YouTubePlaylist | Source::YouTubeChannel) { @@ -398,6 +457,7 @@ fn record_media_entry( hash: &str, file_extension: &str, byte_size: i64, + title: Option, ) -> Result { debug_assert!(run.run_uid.starts_with("run_")); debug_assert!(item.item_uid.starts_with("item_")); @@ -430,7 +490,7 @@ fn record_media_entry( owned_by_user_id: user_id, source_kind: source_kind.to_string(), entity_kind: entity_kind.to_string(), - title: None, + title, visibility: "private".to_string(), representation_kind: representation_kind.to_string(), source_metadata_json: json!({ @@ -458,6 +518,33 @@ fn record_media_entry( Ok(entry) } +/// Extracts PlatformMetadata from a tweet JSON string. +/// Returns Default on any parse failure. +fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata { + let Ok(v) = serde_json::from_str::(json_str) else { + return PlatformMetadata::default(); + }; + + let screen_name = v + .get("author") + .and_then(|a| a.get("screen_name")) + .and_then(|s| s.as_str()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + + let full_text = v + .get("full_text") + .and_then(|t| t.as_str()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + + PlatformMetadata { + author: screen_name, + caption: full_text, + ..Default::default() + } +} + fn record_tweet_entry( conn: &rusqlite::Connection, store_path: &Path, @@ -480,6 +567,12 @@ fn record_tweet_entry( Some(&canonical_locator), &canonical_locator, )?; + // Read tweet JSON early to extract title before entry creation + let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); + let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; + let tweet_meta = tweet_metadata_from_json(&tweet_json); + let tweet_title = generate_entry_title(source, &tweet_meta); + let entry = database::create_archived_entry( conn, &database::NewEntry { @@ -491,7 +584,7 @@ fn record_tweet_entry( owned_by_user_id: user_id, source_kind: source_kind.to_string(), entity_kind: entity_kind.to_string(), - title: None, + title: Some(tweet_title), visibility: "private".to_string(), representation_kind: representation_kind.to_string(), source_metadata_json: json!({ @@ -504,7 +597,6 @@ fn record_tweet_entry( )?; create_structured_root(store_path, &entry)?; - let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); database::add_entry_artifact( conn, &database::NewArtifact { @@ -518,7 +610,6 @@ fn record_tweet_entry( }, )?; - let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? { let raw_path = PathBuf::from(&raw_relpath); let blob = blob_record_for_raw_relpath(store_path, &raw_path)?; @@ -659,6 +750,30 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result = match source { + Source::YouTubeVideo + | Source::X + | Source::Instagram + | Source::Facebook + | Source::TikTok + | Source::Reddit + | Source::Snapchat => downloader::ytdlp::fetch_metadata(&path), + _ => None, + }; + + let local_filename_title: Option = match source { + Source::Local => { + // path is a file:// URI; strip the scheme and take the last component. + let file_path = path.trim_start_matches("file://"); + std::path::Path::new(file_path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + } + _ => None, + }; + let hash = match source { Source::YouTubeVideo | Source::X @@ -728,6 +843,19 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result = if let Some(ref json) = ytdlp_metadata_json { + let metadata = downloader::metadata::extract_from_ytdlp_json(json); + Some(generate_entry_title(source, &metadata)) + } else if let Some(filename) = local_filename_title { + let metadata = PlatformMetadata { + title: Some(filename), + ..Default::default() + }; + Some(generate_entry_title(source, &metadata)) + } else { + None + }; + record_media_entry( &conn, store_path, @@ -740,6 +868,7 @@ pub fn perform_capture(archive_paths: &ArchivePaths, locator: &str) -> Result, + title: Option<&str>, + caption: Option<&str>, + subreddit: Option<&str>, + post_author: Option<&str>, + ) -> PlatformMetadata { + PlatformMetadata { + author: author.map(str::to_string), + title: title.map(str::to_string), + caption: caption.map(str::to_string), + subreddit: subreddit.map(str::to_string), + post_author: post_author.map(str::to_string), + } + } + + #[test] + fn youtube_video_uses_title() { + let m = meta(None, Some("How to Rust"), None, None, None); + assert_eq!(generate_entry_title(Source::YouTubeVideo, &m), "How to Rust"); + } + + #[test] + fn youtube_video_fallback() { + let m = meta(None, None, None, None, None); + assert_eq!(generate_entry_title(Source::YouTubeVideo, &m), "YouTube Video"); + } + + #[test] + fn youtube_playlist_uses_title() { + let m = meta(None, Some("Rust Tutorial Series"), None, None, None); + assert_eq!(generate_entry_title(Source::YouTubePlaylist, &m), "Rust Tutorial Series"); + } + + #[test] + fn youtube_channel_uses_author() { + let m = meta(Some("Rust By Example"), None, None, None, None); + assert_eq!(generate_entry_title(Source::YouTubeChannel, &m), "Archival of Rust By Example"); + } + + #[test] + fn x_media_uses_author() { + let m = meta(Some("alice"), None, None, None, None); + assert_eq!(generate_entry_title(Source::X, &m), "X Media by alice"); + } + + #[test] + fn tweet_uses_excerpt_and_author() { + let m = meta(Some("alice"), None, Some("Hello world"), None, None); + assert_eq!(generate_entry_title(Source::Tweet, &m), "Hello world \u{2014} @alice"); + } + + #[test] + fn tweet_truncates_long_caption() { + let long = "a".repeat(150); + let m = meta(Some("bob"), None, Some(&long), None, None); + let title = generate_entry_title(Source::Tweet, &m); + assert!(title.starts_with(&"a".repeat(100))); + assert!(title.contains("...")); + assert!(title.ends_with("\u{2014} @bob")); + } + + #[test] + fn tweet_thread_uses_author() { + let m = meta(Some("bob"), None, None, None, None); + assert_eq!(generate_entry_title(Source::TweetThread, &m), "Thread by @bob"); + } + + #[test] + fn instagram_uses_author() { + let m = meta(Some("photographer"), None, None, None, None); + assert_eq!(generate_entry_title(Source::Instagram, &m), "Post by @photographer"); + } + + #[test] + fn facebook_uses_author_no_at() { + let m = meta(Some("John Doe"), None, None, None, None); + assert_eq!(generate_entry_title(Source::Facebook, &m), "Post by John Doe"); + } + + #[test] + fn tiktok_uses_author() { + let m = meta(Some("dancemaster"), None, None, None, None); + assert_eq!(generate_entry_title(Source::TikTok, &m), "TikTok by @dancemaster"); + } + + #[test] + fn reddit_full_fields() { + let m = meta(None, Some("My first Rust project"), None, Some("rust"), Some("newbie")); + assert_eq!( + generate_entry_title(Source::Reddit, &m), + "My first Rust project \u{2014} r/rust (u/newbie)" + ); + } + + #[test] + fn snapchat_uses_author() { + let m = meta(Some("snapuser"), None, None, None, None); + assert_eq!(generate_entry_title(Source::Snapchat, &m), "Snap by snapuser"); + } + + #[test] + fn local_uses_title_field() { + let m = meta(None, Some("document.pdf"), None, None, None); + assert_eq!(generate_entry_title(Source::Local, &m), "document.pdf"); + } + + #[test] + fn all_none_falls_back() { + let m = meta(None, None, None, None, None); + assert!(!generate_entry_title(Source::Instagram, &m).is_empty()); + } + + #[test] + fn tweet_title_extracted_from_json() { + let json = r#"{ + "full_text": "Hello Rust world, this is a test tweet", + "author": { "screen_name": "rustacean", "name": "The Rustacean" } + }"#; + let meta = tweet_metadata_from_json(json); + assert_eq!(meta.author, Some("rustacean".to_string())); + assert_eq!(meta.caption, Some("Hello Rust world, this is a test tweet".to_string())); + } + } } diff --git a/crates/archivr-core/src/downloader/metadata.rs b/crates/archivr-core/src/downloader/metadata.rs new file mode 100644 index 0000000..7e4d396 --- /dev/null +++ b/crates/archivr-core/src/downloader/metadata.rs @@ -0,0 +1,110 @@ +use regex::Regex; +use serde_json::Value; +use crate::capture::PlatformMetadata; + +/// Parses a yt-dlp `--dump-json` output string into a PlatformMetadata. +/// Returns a zeroed-out PlatformMetadata on any parse failure — never errors. +pub fn extract_from_ytdlp_json(json_str: &str) -> PlatformMetadata { + let Ok(v): Result = serde_json::from_str(json_str) else { + return PlatformMetadata::default(); + }; + + let str_field = |key: &str| -> Option { + v.get(key) + .and_then(|f| f.as_str()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + }; + + let webpage_url = str_field("webpage_url").unwrap_or_default(); + let uploader = str_field("uploader"); + + // For Reddit, yt-dlp's "uploader" is the post author; subreddit comes from the URL. + // For all other platforms, uploader maps to author. + let subreddit = subreddit_from_url(&webpage_url); + let (author, post_author) = if subreddit.is_some() { + (None, uploader) + } else { + (uploader, None) + }; + + PlatformMetadata { + author, + title: str_field("title"), + caption: str_field("description"), + subreddit, + post_author, + } +} + +fn subreddit_from_url(url: &str) -> Option { + let re = Regex::new(r"reddit\.com/r/([A-Za-z0-9_]+)").expect("static regex is valid"); + re.captures(url) + .and_then(|c| c.get(1)) + .map(|m| m.as_str().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_youtube_fields() { + let json = r#"{ + "uploader": "Tech Channel", + "title": "How Rust Works", + "description": "A deep dive into Rust" + }"#; + let m = extract_from_ytdlp_json(json); + assert_eq!(m.author, Some("Tech Channel".to_string())); + assert_eq!(m.title, Some("How Rust Works".to_string())); + assert_eq!(m.caption, Some("A deep dive into Rust".to_string())); + assert_eq!(m.subreddit, None); + } + + #[test] + fn parses_reddit_subreddit_from_url() { + let json = r#"{ + "uploader": "some_user", + "title": "Cool Post", + "description": "", + "webpage_url": "https://www.reddit.com/r/rust/comments/abc123/cool_post/" + }"#; + let m = extract_from_ytdlp_json(json); + assert_eq!(m.subreddit, Some("rust".to_string())); + assert_eq!(m.post_author, Some("some_user".to_string())); + assert_eq!(m.title, Some("Cool Post".to_string())); + } + + #[test] + fn missing_fields_produce_none() { + let json = r#"{}"#; + let m = extract_from_ytdlp_json(json); + assert!(m.author.is_none()); + assert!(m.title.is_none()); + assert!(m.caption.is_none()); + } + + #[test] + fn invalid_json_returns_default() { + let m = extract_from_ytdlp_json("not json at all"); + assert!(m.author.is_none()); + } + + #[test] + fn empty_string_fields_produce_none() { + let json = r#"{"uploader": "", "title": " ", "description": ""}"#; + let m = extract_from_ytdlp_json(json); + assert_eq!(m.author, None); + assert_eq!(m.title, None); + } + + #[test] + fn subreddit_regex_extracts_name() { + assert_eq!( + subreddit_from_url("https://www.reddit.com/r/rust/comments/abc/"), + Some("rust".to_string()) + ); + assert_eq!(subreddit_from_url("https://youtube.com/watch?v=abc"), None); + } +} diff --git a/crates/archivr-core/src/downloader/mod.rs b/crates/archivr-core/src/downloader/mod.rs index de5d604..d0ba3d8 100644 --- a/crates/archivr-core/src/downloader/mod.rs +++ b/crates/archivr-core/src/downloader/mod.rs @@ -2,3 +2,4 @@ pub mod local; pub mod store; pub mod tweets; pub mod ytdlp; +pub mod metadata; diff --git a/crates/archivr-core/src/downloader/ytdlp.rs b/crates/archivr-core/src/downloader/ytdlp.rs index 6ecd7b8..ba053ee 100644 --- a/crates/archivr-core/src/downloader/ytdlp.rs +++ b/crates/archivr-core/src/downloader/ytdlp.rs @@ -31,3 +31,25 @@ pub fn download(path: String, store_path: &Path, timestamp: &String) -> Result Option { + let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + + let out = std::process::Command::new(&ytdlp) + .arg("--dump-json") + .arg(path) + .output() + .ok()?; + + if !out.status.success() { + return None; + } + + let json = String::from_utf8(out.stdout).ok()?; + if json.trim().is_empty() { None } else { Some(json) } +}