From ccdacfd582362817ff26e33cf1ee34293e9f186d Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:07:00 +0200 Subject: [PATCH] =?UTF-8?q?feat(core):=20YouTube=20playlist/channel/YTM-pl?= =?UTF-8?q?aylist=20capture=20with=20parent=E2=80=93child=20entries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ytdlp: add fetch_playlist_info() using yt-dlp -J --flat-playlist for reliable container title + shallow entry list; normalize item URLs via webpage_url → absolute url → id fallback (domain inferred from container URL so YTM stays on music.youtube.com) - capture: add record_container_entry() (no blob, no primary_media artifact); extend record_media_entry() with parent_entry_id/root_entry_id params (all existing single-item call sites pass None, None) - capture: implement YouTubePlaylist / YouTubeChannel / YouTubeMusicPlaylist capture path replacing the two not-implemented stubs: fetch playlist info → create container entry (reusing existing run + item) → per-child run items (parent_item_id = container item) → download each video/track as a child entry; per-child failures are non-fatal; perform_capture returns result.status reflecting actual run outcome so capture_handler marks the job correctly - archive: add child_count i64 to EntrySummary (col 12 in all listing queries); add get_entry_summary() private helper; fix get_entry_detail() to use get_entry_summary() so child entries are resolvable via the detail endpoint; add list_child_entries(conn, uid, caller_bits) with the same admin/collection visibility predicate as list_root_entries archive_runs.requested_count stays 1 (one user locator); discovered/ completed/failed_count reflect container item + N video items via refresh_run_counters. --- .gitignore | 2 + crates/archivr-core/src/archive.rs | 101 ++++++- crates/archivr-core/src/capture.rs | 265 ++++++++++++++++-- crates/archivr-core/src/downloader/ytdlp.rs | 124 ++++++++ .../plans/2026-07-20-playlist-children.md | 220 +++++++++++++++ 5 files changed, 689 insertions(+), 23 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-20-playlist-children.md diff --git a/.gitignore b/.gitignore index 478a663..eeb041b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ !docs/ !docs/LICENSE !docs/README* +!docs/superpowers/ +!docs/superpowers/** !crates !crates/** diff --git a/crates/archivr-core/src/archive.rs b/crates/archivr-core/src/archive.rs index dd16e66..1a85478 100644 --- a/crates/archivr-core/src/archive.rs +++ b/crates/archivr-core/src/archive.rs @@ -30,6 +30,8 @@ pub struct EntrySummary { pub has_favicon: bool, /// Bytes of blobs already on disk from an earlier entry (precomputed at capture time). pub cached_bytes: i64, + /// Number of direct child entries; 0 for non-container entries. + pub child_count: i64, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] @@ -215,7 +217,8 @@ pub fn list_root_entries( COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, NULL AS parent_entry_uid, EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, - e.cached_bytes + e.cached_bytes, + (SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count FROM archived_entries e JOIN source_identities si ON si.id = e.source_identity_id LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id @@ -248,6 +251,7 @@ pub fn list_root_entries( parent_entry_uid: row.get(9)?, has_favicon: row.get::<_, i64>(10)? != 0, cached_bytes: row.get(11)?, + child_count: row.get(12)?, }) })? .collect::>>()?; @@ -255,6 +259,39 @@ pub fn list_root_entries( Ok(entries) } +/// Fetches one `EntrySummary` for any entry (root or child) by uid. +/// Returns `None` if not found. +fn get_entry_summary( + conn: &rusqlite::Connection, + entry_uid: &str, +) -> Result> { + let sql = format!( + "{} {} WHERE e.entry_uid = ?1 GROUP BY e.id", + ENTRY_SELECT_COLS, ENTRY_FROM_JOINS, + ); + let mut stmt = conn.prepare(&sql)?; + let result = stmt + .query_row([entry_uid], |row| { + Ok(EntrySummary { + entry_uid: row.get(0)?, + archived_at: row.get(1)?, + source_kind: row.get(2)?, + entity_kind: row.get(3)?, + title: row.get(4)?, + visibility: row.get(5)?, + original_url: row.get(6)?, + artifact_count: row.get(7)?, + total_artifact_bytes: row.get(8)?, + parent_entry_uid: row.get(9)?, + has_favicon: row.get::<_, i64>(10)? != 0, + cached_bytes: row.get(11)?, + child_count: row.get(12)?, + }) + }) + .optional()?; + Ok(result) +} + pub fn get_entry_detail( conn: &rusqlite::Connection, entry_uid: &str, @@ -279,9 +316,7 @@ pub fn get_entry_detail( return Ok(None); }; - let summary = list_root_entries(conn, u32::MAX)? - .into_iter() - .find(|entry| entry.entry_uid == entry_uid) + let summary = get_entry_summary(conn, entry_uid)? .context("entry disappeared while loading detail")?; let mut stmt = conn.prepare( @@ -438,12 +473,62 @@ pub fn list_entries_for_collection( parent_entry_uid: row.get(9)?, has_favicon: row.get::<_, i64>(10)? != 0, cached_bytes: row.get(11)?, + child_count: row.get(12)?, }) })? .collect::>>()?; Ok(entries) } +/// Returns the direct children of the entry identified by `parent_entry_uid`, +/// ordered ascending by `archived_at, id` (preserves playlist ordinal feel). +/// Returns an empty vec if the parent has no children or does not exist. +pub fn list_child_entries( + conn: &rusqlite::Connection, + parent_entry_uid: &str, + caller_bits: u32, +) -> Result> { + let sql = format!( + "{} {} \ + WHERE e.parent_entry_id = (SELECT id FROM archived_entries WHERE entry_uid = ?1) \ + AND (\ + CAST(?2 AS INTEGER) & 12 != 0 \ + OR EXISTS (\ + SELECT 1 FROM collection_entries ce \ + WHERE ce.entry_id = e.id \ + AND ce.visibility_bits & CAST(?2 AS INTEGER) != 0\ + )\ + ) \ + GROUP BY e.id \ + ORDER BY e.archived_at ASC, e.id ASC", + ENTRY_SELECT_COLS, ENTRY_FROM_JOINS, + ); + let mut stmt = conn.prepare(&sql)?; + let entries = stmt + .query_map( + rusqlite::params![parent_entry_uid, caller_bits as i64], + |row| { + Ok(EntrySummary { + entry_uid: row.get(0)?, + archived_at: row.get(1)?, + source_kind: row.get(2)?, + entity_kind: row.get(3)?, + title: row.get(4)?, + visibility: row.get(5)?, + original_url: row.get(6)?, + artifact_count: row.get(7)?, + total_artifact_bytes: row.get(8)?, + parent_entry_uid: row.get(9)?, + has_favicon: row.get::<_, i64>(10)? != 0, + cached_bytes: row.get(11)?, + child_count: row.get(12)?, + }) + }, + )? + .collect::>>()?; + Ok(entries) +} + /// Resolves an artifact to its absolute on-disk path under `store_path`. /// /// `artifact.relpath` is a store-relative path (e.g. `raw/a/b/abc.pdf`). @@ -555,7 +640,8 @@ const ENTRY_SELECT_COLS: &str = "SELECT e.entry_uid, e.archived_at, e.source_kin COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \ parent.entry_uid AS parent_entry_uid, \ EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, \ - e.cached_bytes"; + e.cached_bytes, \ + (SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count"; const ENTRY_FROM_JOINS: &str = "FROM archived_entries e \ JOIN source_identities si ON si.id = e.source_identity_id \ @@ -663,6 +749,7 @@ pub fn search_entries( parent_entry_uid: row.get(9)?, has_favicon: row.get::<_, i64>(10)? != 0, cached_bytes: row.get(11)?, + child_count: row.get(12)?, }) })? .collect::>>()?; @@ -870,7 +957,8 @@ pub fn entries_for_tag( COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, parent.entry_uid AS parent_entry_uid, EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, - e.cached_bytes + e.cached_bytes, + (SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count FROM archived_entries e JOIN source_identities si ON si.id = e.source_identity_id LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id @@ -896,6 +984,7 @@ pub fn entries_for_tag( parent_entry_uid: row.get(9)?, has_favicon: row.get::<_, i64>(10)? != 0, cached_bytes: row.get(11)?, + child_count: row.get(12)?, }) })? .collect::>>()?; diff --git a/crates/archivr-core/src/capture.rs b/crates/archivr-core/src/capture.rs index 73fd9bd..1ddcc46 100644 --- a/crates/archivr-core/src/capture.rs +++ b/crates/archivr-core/src/capture.rs @@ -660,6 +660,8 @@ fn record_media_entry( file_extension: &str, byte_size: i64, title: Option, + parent_entry_id: Option, + root_entry_id: Option, ) -> Result { debug_assert!(run.run_uid.starts_with("run_")); debug_assert!(item.item_uid.starts_with("item_")); @@ -686,8 +688,8 @@ fn record_media_entry( &database::NewEntry { source_identity_id, archive_run_id: run.id, - parent_entry_id: None, - root_entry_id: None, + parent_entry_id, + root_entry_id, created_by_user_id: user_id, owned_by_user_id: user_id, source_kind: source_kind.to_string(), @@ -720,6 +722,59 @@ fn record_media_entry( Ok(entry) } +/// Creates a playlist/channel container entry with no blob and no primary-media artifact. +/// Calls `complete_archive_run_item` on the provided item. +fn record_container_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + canonical_locator: &str, + source: Source, + title: Option, + playlist_id: &str, + uploader: Option<&str>, +) -> Result { + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + Some(playlist_id), + Some(canonical_locator), + canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "requested_locator": requested_locator, + "canonical_locator": canonical_locator, + "playlist_id": playlist_id, + "uploader": uploader, + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + /// Extracts PlatformMetadata from a tweet JSON string. /// Returns Default on any parse failure. fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata { @@ -968,14 +1023,191 @@ pub fn perform_capture( )); } - // Sources: YouTube Music Playlist — container expansion not yet implemented. - if source == Source::YouTubeMusicPlaylist { - return Err(fail_run( + // Sources: YouTube playlists, YouTube channels, YouTube Music playlists. + // Fetch container metadata, create a root container entry, then download + // each video/track as a child entry. + // `run` and `item` are already created above; `item` acts as the container item. + if matches!( + source, + Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist + ) { + // `canonical_url` already holds the expanded URL (same value as `path` later). + let playlist_info = match downloader::ytdlp::fetch_playlist_info(&canonical_url, &cookies) { + Ok(info) => info, + Err(e) => { + return Err(fail_run( + &conn, + &run, + &item, + &format!("Failed to fetch playlist info: {e:#}"), + )); + } + }; + + let container_title = playlist_info + .title + .clone() + .or_else(|| playlist_info.uploader.clone().map(|u| format!("{u} (channel)"))); + + let container_entry = match record_container_entry( &conn, + store_path, + user_id, &run, &item, - "YouTube Music playlist archiving is not yet implemented.", - )); + locator, + &canonical_url, + source, + container_title, + &playlist_info.playlist_id, + playlist_info.uploader.as_deref(), + ) { + Ok(e) => e, + Err(e) => { + return Err(fail_run( + &conn, + &run, + &item, + &format!("Failed to create container entry: {e:#}"), + )); + } + }; + + let is_audio = source == Source::YouTubeMusicPlaylist; + let child_quality: Option<&str> = if is_audio { Some("audio") } else { quality }; + let child_entity_kind = if is_audio { "music" } else { "video" }; + + for (ordinal, playlist_item) in playlist_info.items.iter().enumerate() { + let child_timestamp = format!( + "{}-{}", + Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"), + Uuid::new_v4().simple(), + ); + + let child_item = match database::create_archive_run_item( + &conn, + run.id, + Some(item.id), + ordinal as i64, + &playlist_item.url, + Some(&playlist_item.url), + source_kind, + child_entity_kind, + ) { + Ok(i) => i, + Err(e) => { + eprintln!("warn: playlist item {} create_run_item failed: {e:#}", playlist_item.url); + continue; + } + }; + + // Fetch metadata for the child title (best-effort). + let child_meta_json = downloader::ytdlp::fetch_metadata(&playlist_item.url, &cookies); + let child_title: Option = match &child_meta_json { + Some(json) => { + let meta = downloader::metadata::extract_from_ytdlp_json(json); + Some(generate_entry_title( + if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo }, + &meta, + )) + } + None => playlist_item.title.clone(), + }; + + // Download the media. + match downloader::ytdlp::download( + playlist_item.url.clone(), + store_path, + &child_timestamp, + child_quality, + &cookies, + ) { + Ok((hash, file_extension)) => { + let temp_file = store_path + .join("temp") + .join(&child_timestamp) + .join(format!("{child_timestamp}{file_extension}")); + let byte_size = match fs::metadata(&temp_file) { + Ok(m) => m.len() as i64, + Err(e) => { + eprintln!("warn: stat child temp file: {e:#}"); + let _ = database::fail_archive_run_item( + &conn, child_item.id, + &format!("failed to stat downloaded file: {e:#}"), + ); + continue; + } + }; + if !hash_exists(&hash, &file_extension, store_path).unwrap_or(false) { + if let Err(e) = move_temp_to_raw(&temp_file, &hash, store_path) { + eprintln!("warn: move_temp_to_raw child: {e:#}"); + let _ = database::fail_archive_run_item( + &conn, child_item.id, + &format!("failed to move downloaded file: {e:#}"), + ); + continue; + } + } + let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp)); + + let child_source = + if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo }; + match record_media_entry( + &conn, + store_path, + user_id, + &run, + &child_item, + &playlist_item.url, + &playlist_item.url, + child_source, + &hash, + &file_extension, + byte_size, + child_title, + Some(container_entry.id), + Some(container_entry.id), + ) { + Ok(child_entry) => { + let _ = database::refresh_entry_cached_bytes(&conn, child_entry.id); + } + Err(e) => { + eprintln!("warn: record child entry: {e:#}"); + let _ = database::fail_archive_run_item( + &conn, child_item.id, + &format!("failed to record entry: {e:#}"), + ); + } + } + } + Err(e) => { + let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp)); + eprintln!("warn: yt-dlp child download failed for {}: {e:#}", playlist_item.url); + let _ = database::fail_archive_run_item( + &conn, child_item.id, + &format!("yt-dlp download failed: {e:#}"), + ); + } + } + } + + database::refresh_entry_cached_bytes(&conn, container_entry.id)?; + database::finish_archive_run(&conn, run.id)?; + + let run_status: String = conn + .query_row( + "SELECT status FROM archive_runs WHERE id = ?1", + [run.id], + |row| row.get(0), + ) + .unwrap_or_else(|_| "completed".to_string()); + + return Ok(CaptureResult { + run_uid: run.run_uid.clone(), + status: run_status, + ublock_skipped: false, + cookie_ext_skipped: false, + }); } // Source: generic HTTP/S file URL @@ -1018,6 +1250,8 @@ pub fn perform_capture( &file_extension, byte_size, title_hint, + None, + None, )?; database::refresh_entry_cached_bytes(&conn, entry.id)?; database::finish_archive_run(&conn, run.id)?; @@ -1144,8 +1378,10 @@ pub fn perform_capture( &html_hash, &file_extension, byte_size, - entry_title, - )?; + entry_title, + None, + None, + )?; // 5. Add favicon artifact if we captured one. if let Some((fav_relpath, fav_blob_id)) = favicon_info { @@ -1342,14 +1578,7 @@ pub fn perform_capture( } } } - Source::YouTubePlaylist | Source::YouTubeChannel => { - return Err(fail_run( - &conn, - &run, - &item, - "Playlist and channel container expansion are not yet implemented.", - )); - } + Source::YouTubePlaylist | Source::YouTubeChannel => unreachable!(), _ => unreachable!(), }; let temp_file = store_path @@ -1402,6 +1631,8 @@ pub fn perform_capture( &file_extension, byte_size, entry_title, + None, + None, )?; database::refresh_entry_cached_bytes(&conn, media_entry.id)?; database::finish_archive_run(&conn, run.id)?; diff --git a/crates/archivr-core/src/downloader/ytdlp.rs b/crates/archivr-core/src/downloader/ytdlp.rs index 87d4b68..ffdd29e 100644 --- a/crates/archivr-core/src/downloader/ytdlp.rs +++ b/crates/archivr-core/src/downloader/ytdlp.rs @@ -6,10 +6,29 @@ use std::{ process::Command, }; use uuid::Uuid; +use serde_json; use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file}; use crate::hash::hash_file; +/// A single item in a flat playlist listing from `fetch_playlist_info`. +#[derive(Debug)] +pub struct PlaylistItem { + pub id: String, + pub url: String, + pub title: Option, + pub uploader: Option, +} + +/// Container metadata returned by `fetch_playlist_info`. +#[derive(Debug)] +pub struct PlaylistInfo { + pub playlist_id: String, + pub title: Option, + pub uploader: Option, + pub items: Vec, +} + /// Returns the yt-dlp `-f` format selector for `quality`. /// /// - `"audio"` → prefers native Opus/WebM (most efficient), then native @@ -241,6 +260,111 @@ pub fn fetch_metadata(path: &str, cookies: &HashMap) -> Option` and parses the single-JSON result. +/// +/// `-J` / `--dump-single-json` returns one JSON object for the whole +/// container with reliable top-level `title` / `uploader` fields plus an +/// `entries` array of shallow per-item objects. +/// +/// Returns an error if yt-dlp fails, the output is not valid JSON, or +/// the root `_type` is not `"playlist"`. +pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Result { + let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + + let cookie_file: Option = if !cookies.is_empty() { + let domain = domain_from_url(url); + let p = std::env::temp_dir() + .join(format!("archivr-cookies-{}.txt", Uuid::new_v4().simple())); + write_netscape_cookie_file(cookies, &domain, &p) + .context("failed to write yt-dlp cookie file")?; + Some(p) + } else { + None + }; + + let mut cmd = std::process::Command::new(&ytdlp); + cmd.arg("-J").arg("--flat-playlist"); + if let Some(cf) = &cookie_file { + cmd.arg("--cookies").arg(cf); + } + cmd.arg(url); + + let out = cmd.output(); + if let Some(cf) = &cookie_file { + let _ = std::fs::remove_file(cf); + } + let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + bail!("yt-dlp -J --flat-playlist failed for {url}: {stderr}"); + } + + let json: serde_json::Value = serde_json::from_slice(&out.stdout) + .context("yt-dlp -J output is not valid JSON")?; + + let ty = json.get("_type").and_then(|v| v.as_str()).unwrap_or(""); + if ty != "playlist" { + bail!("yt-dlp output _type is {ty:?}, expected \"playlist\""); + } + + let playlist_id = json + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned); + let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned); + + let raw_entries = json + .get("entries") + .and_then(|v| v.as_array()) + .map(|a| a.as_slice()) + .unwrap_or(&[]); + + // Infer the fallback base host from the container URL so YouTube Music + // entries stay on music.youtube.com rather than www.youtube.com. + let fallback_host = if url.contains("music.youtube.com") { + "music.youtube.com" + } else { + "www.youtube.com" + }; + let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://"); + + let mut items = Vec::with_capacity(raw_entries.len()); + for entry in raw_entries { + if entry.is_null() { + continue; // unavailable/private video in flat listing + } + let id = match entry.get("id").and_then(|v| v.as_str()) { + Some(s) => s.to_owned(), + None => continue, + }; + // Normalize the item URL: + // 1. Prefer `webpage_url` — yt-dlp always makes this absolute when present. + // 2. Accept `url` only when it is already an absolute HTTP(S) URL; in + // flat-playlist mode `url` is often just the bare video ID. + // 3. Fallback: construct from `id` using the same host as the container URL. + let item_url = entry + .get("webpage_url") + .and_then(|v| v.as_str()) + .filter(|s| is_absolute(s)) + .map(str::to_owned) + .or_else(|| { + entry + .get("url") + .and_then(|v| v.as_str()) + .filter(|s| is_absolute(s)) + .map(str::to_owned) + }) + .unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}")); + let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned); + let item_uploader = entry.get("uploader").and_then(|v| v.as_str()).map(str::to_owned); + items.push(PlaylistItem { id, url: item_url, title: item_title, uploader: item_uploader }); + } + + Ok(PlaylistInfo { playlist_id, title, uploader, items }) +} + #[cfg(test)] mod tests { use super::{available_video_heights, has_audio_track, quality_format}; diff --git a/docs/superpowers/plans/2026-07-20-playlist-children.md b/docs/superpowers/plans/2026-07-20-playlist-children.md new file mode 100644 index 0000000..fd56f2f --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-playlist-children.md @@ -0,0 +1,220 @@ +# Playlist / Channel Captures + Parent–Child Entry Display + +**Date:** 2026-07-20 +**Branch:** `feat-captures-with-children` + +--- + +## Goal + +1. **YouTube (Music) playlists and channels** — when the user submits a playlist or channel URL/shorthand, archive every video in it as a tree of entries: one root "container" entry for the playlist/channel, one child entry per video. +2. **Parent–child display** — the entry list and detail view can show children of a container entry in a collapsible inline section, replacing the current flat/single-entry-only model. + +--- + +## Schema Contract (no migrations needed — columns already exist) + +| Column | Meaning | +|---|---| +| `archived_entries.parent_entry_id` | `NULL` for root entries; points to container id for children | +| `archived_entries.root_entry_id` | Same as `parent_entry_id` for depth-1 children (direct children of a container) | +| `archive_run_items.parent_item_id` | `NULL` for the container item; points to container item id for video items | +| `archive_runs.requested_count` | Always `1` (user submitted one locator) | +| `archive_runs.discovered_count` | `N + 1` — container item + N video items (from `refresh_run_counters`) | +| `archive_runs.completed_count` | Items completed (container + each successful video) | +| `archive_runs.failed_count` | Items that failed to download | + +**Counter note:** `refresh_run_counters` counts every `archive_run_items` row, so `discovered_count = 1 + N` when a playlist has N videos. This is intentional and accurate — the container item itself counts as a discovered/completed item. + +--- + +## yt-dlp Probe Strategy + +For container sources (playlist, channel, YouTube Music playlist), use: + +```bash +yt-dlp -J --flat-playlist +``` + +`-J` (`--dump-single-json`) returns a **single JSON object** for the whole container — reliable `title` and `uploader` fields at the top level, plus an `entries` array (shallow per-item objects with at minimum `id`, `url`, `title`). This avoids the per-item `playlist_title` reliability issue that `--dump-json` (multi-line) has. + +Individual child video downloads stay single-item (`--no-playlist`), same as current video capture. + +--- + +## File Map + +| File | Action | What changes | +|---|---|---| +| `crates/archivr-core/src/downloader/ytdlp.rs` | Modify | Add `PlaylistItem`, `PlaylistInfo` structs; `fetch_playlist_info()` | +| `crates/archivr-core/src/capture.rs` | Modify | `record_media_entry` gains `parent_entry_id`/`root_entry_id` params; add `record_container_entry()`; implement playlist/channel/YTM-playlist capture paths | +| `crates/archivr-core/src/archive.rs` | Modify | Add `child_count: i64` to `EntrySummary`; add `get_entry_summary()`; fix `get_entry_detail()` to handle child entries; add `list_child_entries()` | +| `crates/archivr-server/src/routes.rs` | Modify | Add `GET /api/archives/:id/entries/:uid/children` route + handler | +| `frontend/src/api.js` | Modify | Add `fetchEntryChildren()` | +| `frontend/src/components/EntryRow.jsx` | Modify | Expand toggle when `child_count > 0`; inline child sub-rows on expand | +| `frontend/src/styles.css` | Modify | Child row indentation + expand toggle styling | + +--- + +## Detailed Design + +### `ytdlp.rs` additions + +```rust +pub struct PlaylistItem { + pub id: String, + pub url: String, + pub title: Option, + pub uploader: Option, +} + +pub struct PlaylistInfo { + pub playlist_id: String, + pub title: Option, + pub uploader: Option, + pub items: Vec, +} + +/// Runs `yt-dlp -J --flat-playlist ` and parses the result. +/// Returns Err if yt-dlp fails or the output isn't a playlist object. +pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Result +``` + +### `capture.rs` changes + +**`record_media_entry` signature extension:** +```rust +fn record_media_entry( + ..., + title: Option, + parent_entry_id: Option, // NEW + root_entry_id: Option, // NEW +) -> Result +``` +All existing call sites pass `None, None`. Playlist child calls pass the container entry's ids. + +**New `record_container_entry()`:** +- Creates an entry with no blob and no `primary_media` artifact. +- `source_kind/entity_kind` from `source_metadata(source)`. +- `parent_entry_id: None, root_entry_id: None`. +- Stores playlist metadata in `source_metadata_json`. +- Returns the `ArchivedEntry` (needed for child `parent_entry_id`). +- Calls `database::complete_archive_run_item()` on the container run item. + +**Playlist/channel capture path (replaces `return Err(...)` stubs):** +``` +1. fetch_playlist_info(url, cookies) → PlaylistInfo +2. create_archive_run(conn, user_id, 1) → run (requested_count=1) +3. create_archive_run_item(run, None, 0, ..., "playlist"/"channel", "container") +4. record_container_entry(...) → container_entry +5. complete_archive_run_item(container_item, container_entry.id) +6. for (ordinal, item) in playlist_info.items: + a. create_archive_run_item(run, Some(container_item.id), ordinal, item.url, ...) + b. fetch_metadata(item.url, cookies) → metadata_json (for title) + c. ytdlp::download(item.url, ...) → (hash, ext) + d. record_media_entry(..., Some(container.id), Some(container.id)) + OR: fail_archive_run_item(child_item, error) and continue +7. finish_archive_run(conn, run.id) +``` + +Error handling: if a child video fails, call `fail_archive_run_item` and continue — partial success is correct for playlists. + +### `archive.rs` changes + +**`EntrySummary` new field:** +```rust +pub child_count: i64, // number of direct children; 0 for non-container entries +``` + +**`ENTRY_SELECT_COLS` extension** (adds col 12): +```sql +(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count +``` + +**`list_root_entries` inline SQL** also extended the same way (col 12). + +All `query_map` closures that build `EntrySummary` get `child_count: row.get(12)?`. + +**`get_entry_summary(conn, entry_uid)`** — new private helper: +- Fetches one entry by uid without the `parent_entry_id IS NULL` constraint. +- Used by the fixed `get_entry_detail`. + +**`get_entry_detail` fix:** +```rust +// BEFORE (broken for child entries): +let summary = list_root_entries(conn, u32::MAX)? + .into_iter() + .find(|entry| entry.entry_uid == entry_uid) + .context("entry disappeared")?; + +// AFTER: +let summary = get_entry_summary(conn, entry_uid)? + .context("entry disappeared")?; +``` + +**New `list_child_entries(conn, parent_uid) -> Result>`:** +```sql +ENTRY_SELECT_COLS ENTRY_FROM_JOINS +WHERE e.parent_entry_id = (SELECT id FROM archived_entries WHERE entry_uid = ?1) +GROUP BY e.id +ORDER BY e.archived_at ASC, e.id ASC +``` +(ascending order — preserves playlist ordinal feel) + +### `routes.rs` addition + +``` +.route( + "/api/archives/:archive_id/entries/:entry_uid/children", + get(list_entry_children), +) +``` + +```rust +async fn list_entry_children( + State(state): State, + auth: AuthUser, + Path((archive_id, entry_uid)): Path<(String, String)>, +) -> Result>, ApiError> { + auth.require_auth()?; + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + Ok(Json(archive::list_child_entries(&conn, &entry_uid)?)) +} +``` + +### Frontend changes + +**`api.js`:** +```js +export async function fetchEntryChildren(archiveId, entryUid) { + return getJson(`/api/archives/${archiveId}/entries/${entryUid}/children`); +} +``` + +**`EntryRow.jsx`:** +- Accept optional `archiveId` for child fetching (already passed). +- When `entry.child_count > 0`: render a chevron expand button in `col-title` area. +- Local state: `expanded` (bool), `children` (array | null), `loading` (bool). +- On chevron click: toggle; if expanding and `children === null`, call `fetchEntryChildren` and store result. +- Render children as `
` containing simplified `` elements (or reuse `EntryRow` without nesting). + +**`styles.css`:** +- `.child-entries` — slight left indent, separator line. +- `.entry-expand-btn` — minimal chevron button. + +--- + +## Acceptance Criteria + +- `cargo test -p archivr-core` green. +- `cargo check -p archivr-server` clean. +- Submitting `yt:playlist/PLxxx` archives the playlist as a container entry with N child video entries in the DB. +- Submitting `yt:@handle` or `yt:channel/UC...` does the same for channel uploads. +- Submitting `ytm:playlist/PLxxx` does the same for YouTube Music playlists. +- Individual video/track captures (`yt:video/ID`, `ytm:ID`) unchanged. +- If one video in a playlist fails, the run status is `failed` but other children are still archived. +- `GET /api/archives/:id/entries` returns root entries only, each with correct `child_count`. +- `GET /api/archives/:id/entries/:uid` works for both root and child entries. +- `GET /api/archives/:id/entries/:uid/children` returns the child entries for a container. +- Frontend expand button appears on container entries; clicking it fetches and shows children inline.