From 814aa76a1d92897dcae70c4ae6be89525574665f Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:34:10 +0200 Subject: [PATCH] feat(core): playlist per-item quality + incremental sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ytdlp.rs: - Add PlaylistItemProbe / PlaylistProbeResult (pub, serde::Serialize) - Add private available_video_heights_from_value() helper for Value entries - Add probe_playlist_qualities(): yt-dlp -J (full metadata, no flat flag) returns per-video quality lists in one subprocess call capture.rs: - Add per_item_quality: HashMap and sync: bool to CaptureConfig (both Default; keyed by yt-dlp video ID, not URL) - Add pub locator_to_playlist_url(): validates only the three playlist sources, expands shorthands; keeps locator_to_ytdlp_url's no-playlists contract - Playlist capture block: sync-aware container resolution - sync + existing container → reuse it via complete_archive_run_item, skip already-archived children (by canonical URL) before creating run items so refresh_run_counters only counts new items - sync + no container → create normally (first sync run) - non-sync → always create fresh container (existing behaviour) - Per-item quality: config.per_item_quality.get(id) falls back to child_quality archive.rs: - Add get_archived_playlist_child_urls(): returns HashSet of canonical URLs of all children under any container matching the playlist canonical URL - Add find_container_entry_id_by_canonical_url(): returns most-recent container entry id (parent_entry_id IS NULL) for a given canonical URL routes.rs: stub per_item_quality/sync on both CaptureConfig sites (server agent will wire body fields in Phase 2) --- crates/archivr-core/src/archive.rs | 53 ++++++++ crates/archivr-core/src/capture.rs | 116 +++++++++++++---- crates/archivr-core/src/downloader/ytdlp.rs | 130 ++++++++++++++++++++ crates/archivr-server/src/routes.rs | 4 + 4 files changed, 277 insertions(+), 26 deletions(-) diff --git a/crates/archivr-core/src/archive.rs b/crates/archivr-core/src/archive.rs index 1a85478..40d67a2 100644 --- a/crates/archivr-core/src/archive.rs +++ b/crates/archivr-core/src/archive.rs @@ -529,6 +529,59 @@ pub fn list_child_entries( Ok(entries) } +/// Returns the set of canonical URLs for all child entries archived under +/// **any** container entry whose own canonical URL matches `playlist_canonical_url`. +/// +/// Used in sync mode so the playlist capture path can skip videos that were +/// already downloaded in a previous run of the same playlist. +pub fn get_archived_playlist_child_urls( + conn: &rusqlite::Connection, + playlist_canonical_url: &str, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT si_child.canonical_url \ + FROM archived_entries child \ + JOIN source_identities si_child ON si_child.id = child.source_identity_id \ + WHERE child.parent_entry_id IN ( \ + SELECT e.id \ + FROM archived_entries e \ + JOIN source_identities si ON si.id = e.source_identity_id \ + WHERE si.canonical_url = ?1 \ + AND e.parent_entry_id IS NULL \ + )", + )?; + let urls = stmt + .query_map([playlist_canonical_url], |row| row.get::<_, String>(0))? + .filter_map(|r| r.ok()) + .collect::>(); + Ok(urls) +} + +/// Finds the most recent container entry (parent_entry_id IS NULL) whose +/// canonical URL matches `canonical_url`. Returns its row id, or None if +/// no such entry exists. +/// +/// Used in sync mode to reuse an existing playlist/channel container instead +/// of creating a duplicate root entry on every sync run. +pub fn find_container_entry_id_by_canonical_url( + conn: &rusqlite::Connection, + canonical_url: &str, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT e.id \ + FROM archived_entries e \ + JOIN source_identities si ON si.id = e.source_identity_id \ + WHERE si.canonical_url = ?1 \ + AND e.parent_entry_id IS NULL \ + ORDER BY e.archived_at DESC \ + LIMIT 1", + )?; + let id = stmt + .query_row([canonical_url], |row| row.get::<_, i64>(0)) + .optional()?; + Ok(id) +} + /// Resolves an artifact to its absolute on-disk path under `store_path`. /// /// `artifact.relpath` is a store-relative path (e.g. `raw/a/b/abc.pdf`). diff --git a/crates/archivr-core/src/capture.rs b/crates/archivr-core/src/capture.rs index 1ddcc46..39fdfb0 100644 --- a/crates/archivr-core/src/capture.rs +++ b/crates/archivr-core/src/capture.rs @@ -7,7 +7,7 @@ use std::{ fs, path::{Path, PathBuf}, }; -use crate::{archive::ArchivePaths, database, downloader, twitter::parse_tweet_id}; +use crate::{archive::{self, ArchivePaths}, database, downloader, twitter::parse_tweet_id}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Source { @@ -90,6 +90,14 @@ pub struct CaptureConfig { /// Route WebPage captures through the Freedium mirror to bypass paywalls. /// The original locator is still recorded in the DB; only the fetch URL changes. pub via_freedium: bool, + /// Per-item quality overrides for playlist captures. + /// Keys are yt-dlp video IDs (e.g. "dQw4w9WgXcQ"); values are quality strings + /// accepted by `quality_format` ("best", "audio", "1080p", etc.). + /// Empty map = apply the global `quality` to all items. + pub per_item_quality: HashMap, + /// When true, skip playlist items whose URL is already archived as a child + /// of any container entry with the same canonical playlist URL. + pub sync: bool, } /// Resolves which cookies apply to `url` by evaluating all rules in ordinal order. @@ -536,6 +544,19 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option { } } +/// Returns the canonical URL for playlist/channel locators that yt-dlp can +/// expand with `--flat-playlist`, or `None` for non-playlist sources. +/// Intentionally separate from `locator_to_ytdlp_url`, which excludes playlists. +pub fn locator_to_playlist_url(locator: &str) -> Option { + let source = determine_source(locator); + match source { + Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist => { + Some(expand_shorthand_to_url(locator, &source)) + } + _ => None, + } +} + fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result { let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); Ok(path.exists()) @@ -1049,28 +1070,58 @@ pub fn perform_capture( .clone() .or_else(|| playlist_info.uploader.clone().map(|u| format!("{u} (channel)"))); - let container_entry = match record_container_entry( - &conn, - store_path, - user_id, - &run, - &item, - locator, - &canonical_url, - source, - container_title, - &playlist_info.playlist_id, - playlist_info.uploader.as_deref(), - ) { - Ok(e) => e, - Err(e) => { - return Err(fail_run( - &conn, - &run, - &item, - &format!("Failed to create container entry: {e:#}"), - )); + // Sync mode: reuse an existing container so we don't create a duplicate + // root entry on every sync run. Non-sync always creates a fresh container. + let (container_id, already_archived) = if config.sync { + match archive::find_container_entry_id_by_canonical_url(&conn, &canonical_url) { + Err(e) => { + return Err(fail_run( + &conn, &run, &item, + &format!("Failed to query existing container: {e:#}"), + )); + } + Ok(Some(existing_id)) => { + // Container already exists — mark the run item done pointing at it; + // don't call record_container_entry (that would create a duplicate). + if let Err(e) = database::complete_archive_run_item(&conn, item.id, existing_id) { + return Err(fail_run( + &conn, &run, &item, + &format!("Failed to complete run item for existing container: {e:#}"), + )); + } + let archived = archive::get_archived_playlist_child_urls(&conn, &canonical_url) + .unwrap_or_default(); + (existing_id, archived) + } + Ok(None) => { + // First sync run for this playlist — create the container normally. + let e = match record_container_entry( + &conn, store_path, user_id, &run, &item, locator, &canonical_url, + source, container_title, &playlist_info.playlist_id, + playlist_info.uploader.as_deref(), + ) { + Ok(e) => e, + Err(e) => return Err(fail_run( + &conn, &run, &item, + &format!("Failed to create container entry: {e:#}"), + )), + }; + (e.id, std::collections::HashSet::new()) + } } + } else { + let e = match record_container_entry( + &conn, store_path, user_id, &run, &item, locator, &canonical_url, + source, container_title, &playlist_info.playlist_id, + playlist_info.uploader.as_deref(), + ) { + Ok(e) => e, + Err(e) => return Err(fail_run( + &conn, &run, &item, + &format!("Failed to create container entry: {e:#}"), + )), + }; + (e.id, std::collections::HashSet::new()) }; let is_audio = source == Source::YouTubeMusicPlaylist; @@ -1078,6 +1129,12 @@ pub fn perform_capture( let child_entity_kind = if is_audio { "music" } else { "video" }; for (ordinal, playlist_item) in playlist_info.items.iter().enumerate() { + // Sync: skip items already archived — no run item created, so + // refresh_run_counters sees only newly-attempted items. + if config.sync && already_archived.contains(&playlist_item.url) { + continue; + } + let child_timestamp = format!( "{}-{}", Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"), @@ -1114,12 +1171,19 @@ pub fn perform_capture( None => playlist_item.title.clone(), }; + // Per-item quality override keyed by yt-dlp video ID; fall back to playlist-level. + let effective_child_quality: Option<&str> = config + .per_item_quality + .get(&playlist_item.id) + .map(String::as_str) + .or(child_quality); + // Download the media. match downloader::ytdlp::download( playlist_item.url.clone(), store_path, &child_timestamp, - child_quality, + effective_child_quality, &cookies, ) { Ok((hash, file_extension)) => { @@ -1165,8 +1229,8 @@ pub fn perform_capture( &file_extension, byte_size, child_title, - Some(container_entry.id), - Some(container_entry.id), + Some(container_id), + Some(container_id), ) { Ok(child_entry) => { let _ = database::refresh_entry_cached_bytes(&conn, child_entry.id); @@ -1191,7 +1255,7 @@ pub fn perform_capture( } } - database::refresh_entry_cached_bytes(&conn, container_entry.id)?; + database::refresh_entry_cached_bytes(&conn, container_id)?; database::finish_archive_run(&conn, run.id)?; let run_status: String = conn diff --git a/crates/archivr-core/src/downloader/ytdlp.rs b/crates/archivr-core/src/downloader/ytdlp.rs index ffdd29e..5f6423f 100644 --- a/crates/archivr-core/src/downloader/ytdlp.rs +++ b/crates/archivr-core/src/downloader/ytdlp.rs @@ -29,6 +29,27 @@ pub struct PlaylistInfo { pub items: Vec, } +/// Per-item quality data returned by `probe_playlist_qualities`. +#[derive(Debug, serde::Serialize)] +pub struct PlaylistItemProbe { + pub id: String, + pub url: String, + pub title: Option, + /// Available video heights as strings (e.g. "1080p"), sorted highest-first. + /// Empty vec means audio-only (no video track). + pub qualities: Vec, + pub has_audio: bool, +} + +/// Full playlist probe result with per-item quality data. +#[derive(Debug, serde::Serialize)] +pub struct PlaylistProbeResult { + pub playlist_id: String, + pub title: Option, + pub uploader: Option, + pub items: Vec, +} + /// Returns the yt-dlp `-f` format selector for `quality`. /// /// - `"audio"` → prefers native Opus/WebM (most efficient), then native @@ -93,6 +114,23 @@ pub fn available_video_heights(json: &str) -> Vec { heights } +/// Extracts distinct video heights from a serde_json entry Value's `formats` array, +/// sorted highest-first. Audio-only formats (vcodec == "none") are excluded. +fn available_video_heights_from_value(entry: &serde_json::Value) -> Vec { + let Some(fmts) = entry.get("formats").and_then(|v| v.as_array()) else { + return vec![]; + }; + let mut heights: Vec = fmts + .iter() + .filter(|f| f.get("vcodec").and_then(|c| c.as_str()).unwrap_or("none") != "none") + .filter_map(|f| f.get("height").and_then(|h| h.as_u64()).map(|h| h as u32)) + .filter(|&h| h > 0) + .collect(); + heights.sort_unstable_by(|a, b| b.cmp(a)); + heights.dedup(); + heights +} + /// Returns true when the yt-dlp `--dump-json` response contains at least one /// format with a real audio codec (i.e. `acodec != "none"`). pub fn has_audio_track(json: &str) -> bool { @@ -365,6 +403,98 @@ pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Resu Ok(PlaylistInfo { playlist_id, title, uploader, items }) } +/// Runs `yt-dlp -J ` (full metadata, NOT --flat-playlist) and returns +/// per-item quality data for every entry in the playlist. +/// +/// This makes one yt-dlp subprocess call that fetches full format data for +/// all videos — expensive for large playlists but gives accurate per-video +/// quality lists. Intended for pre-capture quality selection only. +pub fn probe_playlist_qualities( + url: &str, + cookies: &HashMap, +) -> Result { + let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + + let cookie_file: Option = if !cookies.is_empty() { + let domain = domain_from_url(url); + let p = std::env::temp_dir() + .join(format!("archivr-cookies-{}.txt", Uuid::new_v4().simple())); + write_netscape_cookie_file(cookies, &domain, &p) + .context("failed to write yt-dlp cookie file")?; + Some(p) + } else { + None + }; + + let mut cmd = std::process::Command::new(&ytdlp); + cmd.arg("-J"); // full metadata — NOT --flat-playlist + if let Some(cf) = &cookie_file { + cmd.arg("--cookies").arg(cf); + } + cmd.arg(url); + + let out = cmd.output(); + if let Some(cf) = &cookie_file { + let _ = std::fs::remove_file(cf); + } + let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + bail!("yt-dlp -J failed for {url}: {stderr}"); + } + + let json: serde_json::Value = serde_json::from_slice(&out.stdout) + .context("yt-dlp -J output is not valid JSON")?; + + let ty = json.get("_type").and_then(|v| v.as_str()).unwrap_or(""); + if ty != "playlist" { + bail!("yt-dlp output _type is {ty:?}, expected \"playlist\""); + } + + let playlist_id = json.get("id").and_then(|v| v.as_str()).unwrap_or("").to_string(); + let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned); + let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned); + + // Same URL normalization as fetch_playlist_info + let fallback_host = if url.contains("music.youtube.com") { + "music.youtube.com" + } else { + "www.youtube.com" + }; + let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://"); + + let raw_entries = json + .get("entries") + .and_then(|v| v.as_array()) + .map(|a| a.as_slice()) + .unwrap_or(&[]); + + let mut items = Vec::with_capacity(raw_entries.len()); + for entry in raw_entries { + if entry.is_null() { continue; } + let id = match entry.get("id").and_then(|v| v.as_str()) { + Some(s) => s.to_owned(), + None => continue, + }; + let item_url = entry + .get("webpage_url").and_then(|v| v.as_str()).filter(|s| is_absolute(s)).map(str::to_owned) + .or_else(|| entry.get("url").and_then(|v| v.as_str()).filter(|s| is_absolute(s)).map(str::to_owned)) + .unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}")); + let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned); + let heights = available_video_heights_from_value(entry); + let qualities: Vec = heights.iter().map(|h| format!("{h}p")).collect(); + let has_audio = entry + .get("formats").and_then(|v| v.as_array()) + .map(|fmts| fmts.iter().any(|f| { + f.get("acodec").and_then(|c| c.as_str()).unwrap_or("none") != "none" + })) + .unwrap_or(false); + items.push(PlaylistItemProbe { id, url: item_url, title: item_title, qualities, has_audio }); + } + + Ok(PlaylistProbeResult { playlist_id, title, uploader, items }) +} + #[cfg(test)] mod tests { use super::{available_video_heights, has_audio_track, quality_format}; diff --git a/crates/archivr-server/src/routes.rs b/crates/archivr-server/src/routes.rs index a06af9a..6ff3a5b 100644 --- a/crates/archivr-server/src/routes.rs +++ b/crates/archivr-server/src/routes.rs @@ -879,6 +879,8 @@ async fn capture_handler( modal_closer_enabled: Some(effective_modal_closer), reader_mode: body.reader_mode.unwrap_or(false), via_freedium: body.via_freedium.unwrap_or(true), + per_item_quality: std::collections::HashMap::new(), + sync: false, }; // Spawn background capture. @@ -998,6 +1000,8 @@ async fn rearchive_handler( modal_closer_enabled: None, reader_mode: false, via_freedium: false, + per_item_quality: std::collections::HashMap::new(), + sync: false, }; let job_uid_bg = job_uid.clone();