thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00
Code Issues Activity

feat(core): YouTube playlist/channel/YTM-playlist capture with parent–child entries

Browse source 
- ytdlp: add fetch_playlist_info() using yt-dlp -J --flat-playlist for
  reliable container title + shallow entry list; normalize item URLs via
  webpage_url → absolute url → id fallback (domain inferred from container
  URL so YTM stays on music.youtube.com)
- capture: add record_container_entry() (no blob, no primary_media artifact);
  extend record_media_entry() with parent_entry_id/root_entry_id params (all
  existing single-item call sites pass None, None)
- capture: implement YouTubePlaylist / YouTubeChannel / YouTubeMusicPlaylist
  capture path replacing the two not-implemented stubs: fetch playlist info →
  create container entry (reusing existing run + item) → per-child run items
  (parent_item_id = container item) → download each video/track as a child
  entry; per-child failures are non-fatal; perform_capture returns result.status
  reflecting actual run outcome so capture_handler marks the job correctly
- archive: add child_count i64 to EntrySummary (col 12 in all listing queries);
  add get_entry_summary() private helper; fix get_entry_detail() to use
  get_entry_summary() so child entries are resolvable via the detail endpoint;
  add list_child_entries(conn, uid, caller_bits) with the same
  admin/collection visibility predicate as list_root_entries

archive_runs.requested_count stays 1 (one user locator); discovered/
completed/failed_count reflect container item + N video items via
refresh_run_counters.
This commit is contained in:
TheGeneralist 2026-07-20 16:07:00 +02:00
parent d202e177e1
commit ccdacfd582
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
5 changed files with 689 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -7,6 +7,8 @@
!docs/
!docs/LICENSE
!docs/README*
!docs/superpowers/
!docs/superpowers/**
!crates
!crates/**

101
crates/archivr-core/src/archive.rs
View file

@ -30,6 +30,8 @@ pub struct EntrySummary {
pub has_favicon: bool,
/// Bytes of blobs already on disk from an earlier entry (precomputed at capture time).
pub cached_bytes: i64,
/// Number of direct child entries; 0 for non-container entries.
pub child_count: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
@ -215,7 +217,8 @@ pub fn list_root_entries(
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
NULL AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon,
e.cached_bytes
e.cached_bytes,
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count
FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -248,6 +251,7 @@ pub fn list_root_entries(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -255,6 +259,39 @@ pub fn list_root_entries(
Ok(entries)
}
/// Fetches one `EntrySummary` for any entry (root or child) by uid.
/// Returns `None` if not found.
fn get_entry_summary(
conn: &rusqlite::Connection,
entry_uid: &str,
) -> Result<Option<EntrySummary>> {
let sql = format!(
"{} {} WHERE e.entry_uid = ?1 GROUP BY e.id",
ENTRY_SELECT_COLS, ENTRY_FROM_JOINS,
);
let mut stmt = conn.prepare(&sql)?;
let result = stmt
.query_row([entry_uid], |row| {
Ok(EntrySummary {
entry_uid: row.get(0)?,
archived_at: row.get(1)?,
source_kind: row.get(2)?,
entity_kind: row.get(3)?,
title: row.get(4)?,
visibility: row.get(5)?,
original_url: row.get(6)?,
artifact_count: row.get(7)?,
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})
.optional()?;
Ok(result)
}
pub fn get_entry_detail(
conn: &rusqlite::Connection,
entry_uid: &str,
@ -279,9 +316,7 @@ pub fn get_entry_detail(
return Ok(None);
};
let summary = list_root_entries(conn, u32::MAX)?
.into_iter()
.find(|entry| entry.entry_uid == entry_uid)
let summary = get_entry_summary(conn, entry_uid)?
.context("entry disappeared while loading detail")?;
let mut stmt = conn.prepare(
@ -438,12 +473,62 @@ pub fn list_entries_for_collection(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(entries)
}
/// Returns the direct children of the entry identified by `parent_entry_uid`,
/// ordered ascending by `archived_at, id` (preserves playlist ordinal feel).
/// Returns an empty vec if the parent has no children or does not exist.
pub fn list_child_entries(
conn: &rusqlite::Connection,
parent_entry_uid: &str,
caller_bits: u32,
) -> Result<Vec<EntrySummary>> {
let sql = format!(
"{} {} \
WHERE e.parent_entry_id = (SELECT id FROM archived_entries WHERE entry_uid = ?1) \
AND (\
CAST(?2 AS INTEGER) & 12 != 0 \
OR EXISTS (\
SELECT 1 FROM collection_entries ce \
WHERE ce.entry_id = e.id \
AND ce.visibility_bits & CAST(?2 AS INTEGER) != 0\
)\
) \
GROUP BY e.id \
ORDER BY e.archived_at ASC, e.id ASC",
ENTRY_SELECT_COLS, ENTRY_FROM_JOINS,
);
let mut stmt = conn.prepare(&sql)?;
let entries = stmt
.query_map(
rusqlite::params![parent_entry_uid, caller_bits as i64],
|row| {
Ok(EntrySummary {
entry_uid: row.get(0)?,
archived_at: row.get(1)?,
source_kind: row.get(2)?,
entity_kind: row.get(3)?,
title: row.get(4)?,
visibility: row.get(5)?,
original_url: row.get(6)?,
artifact_count: row.get(7)?,
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
},
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(entries)
}
/// Resolves an artifact to its absolute on-disk path under `store_path`.
///
/// `artifact.relpath` is a store-relative path (e.g. `raw/a/b/abc.pdf`).
@ -555,7 +640,8 @@ const ENTRY_SELECT_COLS: &str = "SELECT e.entry_uid, e.archived_at, e.source_kin
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \
parent.entry_uid AS parent_entry_uid, \
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, \
e.cached_bytes";
e.cached_bytes, \
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count";
const ENTRY_FROM_JOINS: &str = "FROM archived_entries e \
JOIN source_identities si ON si.id = e.source_identity_id \
@ -663,6 +749,7 @@ pub fn search_entries(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -870,7 +957,8 @@ pub fn entries_for_tag(
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
parent.entry_uid AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon,
e.cached_bytes
e.cached_bytes,
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count
FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -896,6 +984,7 @@ pub fn entries_for_tag(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;

265
crates/archivr-core/src/capture.rs
View file

@ -660,6 +660,8 @@ fn record_media_entry(
file_extension: &str,
byte_size: i64,
title: Option<String>,
parent_entry_id: Option<i64>,
root_entry_id: Option<i64>,
) -> Result<database::ArchivedEntry> {
debug_assert!(run.run_uid.starts_with("run_"));
debug_assert!(item.item_uid.starts_with("item_"));
@ -686,8 +688,8 @@ fn record_media_entry(
&database::NewEntry {
source_identity_id,
archive_run_id: run.id,
parent_entry_id: None,
root_entry_id: None,
parent_entry_id,
root_entry_id,
created_by_user_id: user_id,
owned_by_user_id: user_id,
source_kind: source_kind.to_string(),
@ -720,6 +722,59 @@ fn record_media_entry(
Ok(entry)
}
/// Creates a playlist/channel container entry with no blob and no primary-media artifact.
/// Calls `complete_archive_run_item` on the provided item.
fn record_container_entry(
conn: &rusqlite::Connection,
store_path: &Path,
user_id: i64,
run: &database::ArchiveRun,
item: &database::ArchiveRunItem,
requested_locator: &str,
canonical_locator: &str,
source: Source,
title: Option<String>,
playlist_id: &str,
uploader: Option<&str>,
) -> Result<database::ArchivedEntry> {
let (source_kind, entity_kind, representation_kind) = source_metadata(source);
let source_identity_id = database::upsert_source_identity(
conn,
source_kind,
entity_kind,
Some(playlist_id),
Some(canonical_locator),
canonical_locator,
)?;
let entry = database::create_archived_entry(
conn,
&database::NewEntry {
source_identity_id,
archive_run_id: run.id,
parent_entry_id: None,
root_entry_id: None,
created_by_user_id: user_id,
owned_by_user_id: user_id,
source_kind: source_kind.to_string(),
entity_kind: entity_kind.to_string(),
title,
visibility: "private".to_string(),
representation_kind: representation_kind.to_string(),
source_metadata_json: json!({
"requested_locator": requested_locator,
"canonical_locator": canonical_locator,
"playlist_id": playlist_id,
"uploader": uploader,
})
.to_string(),
display_metadata_json: None,
},
)?;
create_structured_root(store_path, &entry)?;
database::complete_archive_run_item(conn, item.id, entry.id)?;
Ok(entry)
}
/// Extracts PlatformMetadata from a tweet JSON string.
/// Returns Default on any parse failure.
fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
@ -968,14 +1023,191 @@ pub fn perform_capture(
));
}
// Sources: YouTube Music Playlist — container expansion not yet implemented.
if source == Source::YouTubeMusicPlaylist {
return Err(fail_run(
// Sources: YouTube playlists, YouTube channels, YouTube Music playlists.
// Fetch container metadata, create a root container entry, then download
// each video/track as a child entry.
// `run` and `item` are already created above; `item` acts as the container item.
if matches!(
source,
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist
) {
// `canonical_url` already holds the expanded URL (same value as `path` later).
let playlist_info = match downloader::ytdlp::fetch_playlist_info(&canonical_url, &cookies) {
Ok(info) => info,
Err(e) => {
return Err(fail_run(
&conn,
&run,
&item,
&format!("Failed to fetch playlist info: {e:#}"),
));
}
};
let container_title = playlist_info
.title
.clone()
.or_else(|| playlist_info.uploader.clone().map(|u| format!("{u} (channel)")));
let container_entry = match record_container_entry(
&conn,
store_path,
user_id,
&run,
&item,
"YouTube Music playlist archiving is not yet implemented.",
));
locator,
&canonical_url,
source,
container_title,
&playlist_info.playlist_id,
playlist_info.uploader.as_deref(),
) {
Ok(e) => e,
Err(e) => {
return Err(fail_run(
&conn,
&run,
&item,
&format!("Failed to create container entry: {e:#}"),
));
}
};
let is_audio = source == Source::YouTubeMusicPlaylist;
let child_quality: Option<&str> = if is_audio { Some("audio") } else { quality };
let child_entity_kind = if is_audio { "music" } else { "video" };
for (ordinal, playlist_item) in playlist_info.items.iter().enumerate() {
let child_timestamp = format!(
"{}-{}",
Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
Uuid::new_v4().simple(),
);
let child_item = match database::create_archive_run_item(
&conn,
run.id,
Some(item.id),
ordinal as i64,
&playlist_item.url,
Some(&playlist_item.url),
source_kind,
child_entity_kind,
) {
Ok(i) => i,
Err(e) => {
eprintln!("warn: playlist item {} create_run_item failed: {e:#}", playlist_item.url);
continue;
}
};
// Fetch metadata for the child title (best-effort).
let child_meta_json = downloader::ytdlp::fetch_metadata(&playlist_item.url, &cookies);
let child_title: Option<String> = match &child_meta_json {
Some(json) => {
let meta = downloader::metadata::extract_from_ytdlp_json(json);
Some(generate_entry_title(
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo },
&meta,
))
}
None => playlist_item.title.clone(),
};
// Download the media.
match downloader::ytdlp::download(
playlist_item.url.clone(),
store_path,
&child_timestamp,
child_quality,
&cookies,
) {
Ok((hash, file_extension)) => {
let temp_file = store_path
.join("temp")
.join(&child_timestamp)
.join(format!("{child_timestamp}{file_extension}"));
let byte_size = match fs::metadata(&temp_file) {
Ok(m) => m.len() as i64,
Err(e) => {
eprintln!("warn: stat child temp file: {e:#}");
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("failed to stat downloaded file: {e:#}"),
);
continue;
}
};
if !hash_exists(&hash, &file_extension, store_path).unwrap_or(false) {
if let Err(e) = move_temp_to_raw(&temp_file, &hash, store_path) {
eprintln!("warn: move_temp_to_raw child: {e:#}");
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("failed to move downloaded file: {e:#}"),
);
continue;
}
}
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
let child_source =
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo };
match record_media_entry(
&conn,
store_path,
user_id,
&run,
&child_item,
&playlist_item.url,
&playlist_item.url,
child_source,
&hash,
&file_extension,
byte_size,
child_title,
Some(container_entry.id),
Some(container_entry.id),
) {
Ok(child_entry) => {
let _ = database::refresh_entry_cached_bytes(&conn, child_entry.id);
}
Err(e) => {
eprintln!("warn: record child entry: {e:#}");
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("failed to record entry: {e:#}"),
);
}
}
}
Err(e) => {
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
eprintln!("warn: yt-dlp child download failed for {}: {e:#}", playlist_item.url);
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("yt-dlp download failed: {e:#}"),
);
}
}
}
database::refresh_entry_cached_bytes(&conn, container_entry.id)?;
database::finish_archive_run(&conn, run.id)?;
let run_status: String = conn
.query_row(
"SELECT status FROM archive_runs WHERE id = ?1",
[run.id],
|row| row.get(0),
)
.unwrap_or_else(|_| "completed".to_string());
return Ok(CaptureResult {
run_uid: run.run_uid.clone(),
status: run_status,
ublock_skipped: false,
cookie_ext_skipped: false,
});
}
// Source: generic HTTP/S file URL
@ -1018,6 +1250,8 @@ pub fn perform_capture(
&file_extension,
byte_size,
title_hint,
None,
None,
)?;
database::refresh_entry_cached_bytes(&conn, entry.id)?;
database::finish_archive_run(&conn, run.id)?;
@ -1144,8 +1378,10 @@ pub fn perform_capture(
&html_hash,
&file_extension,
byte_size,
entry_title,
)?;
entry_title,
None,
None,
)?;
// 5. Add favicon artifact if we captured one.
if let Some((fav_relpath, fav_blob_id)) = favicon_info {
@ -1342,14 +1578,7 @@ pub fn perform_capture(
}
}
}
Source::YouTubePlaylist | Source::YouTubeChannel => {
return Err(fail_run(
&conn,
&run,
&item,
"Playlist and channel container expansion are not yet implemented.",
));
}
Source::YouTubePlaylist | Source::YouTubeChannel => unreachable!(),
_ => unreachable!(),
};
let temp_file = store_path
@ -1402,6 +1631,8 @@ pub fn perform_capture(
&file_extension,
byte_size,
entry_title,
None,
None,
)?;
database::refresh_entry_cached_bytes(&conn, media_entry.id)?;
database::finish_archive_run(&conn, run.id)?;

124
crates/archivr-core/src/downloader/ytdlp.rs
View file

@ -6,10 +6,29 @@ use std::{
process::Command,
};
use uuid::Uuid;
use serde_json;
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
use crate::hash::hash_file;
/// A single item in a flat playlist listing from `fetch_playlist_info`.
#[derive(Debug)]
pub struct PlaylistItem {
pub id: String,
pub url: String,
pub title: Option<String>,
pub uploader: Option<String>,
}
/// Container metadata returned by `fetch_playlist_info`.
#[derive(Debug)]
pub struct PlaylistInfo {
pub playlist_id: String,
pub title: Option<String>,
pub uploader: Option<String>,
pub items: Vec<PlaylistItem>,
}
/// Returns the yt-dlp `-f` format selector for `quality`.
///
/// - `"audio"` → prefers native Opus/WebM (most efficient), then native
@ -241,6 +260,111 @@ pub fn fetch_metadata(path: &str, cookies: &HashMap<String, String>) -> Option<S
if json.trim().is_empty() { None } else { Some(json) }
}
/// Runs `yt-dlp -J --flat-playlist <url>` and parses the single-JSON result.
///
/// `-J` / `--dump-single-json` returns one JSON object for the whole
/// container with reliable top-level `title` / `uploader` fields plus an
/// `entries` array of shallow per-item objects.
///
/// Returns an error if yt-dlp fails, the output is not valid JSON, or
/// the root `_type` is not `"playlist"`.
pub fn fetch_playlist_info(url: &str, cookies: &HashMap<String, String>) -> Result<PlaylistInfo> {
let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
let domain = domain_from_url(url);
let p = std::env::temp_dir()
.join(format!("archivr-cookies-{}.txt", Uuid::new_v4().simple()));
write_netscape_cookie_file(cookies, &domain, &p)
.context("failed to write yt-dlp cookie file")?;
Some(p)
} else {
None
};
let mut cmd = std::process::Command::new(&ytdlp);
cmd.arg("-J").arg("--flat-playlist");
if let Some(cf) = &cookie_file {
cmd.arg("--cookies").arg(cf);
}
cmd.arg(url);
let out = cmd.output();
if let Some(cf) = &cookie_file {
let _ = std::fs::remove_file(cf);
}
let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?;
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr);
bail!("yt-dlp -J --flat-playlist failed for {url}: {stderr}");
}
let json: serde_json::Value = serde_json::from_slice(&out.stdout)
.context("yt-dlp -J output is not valid JSON")?;
let ty = json.get("_type").and_then(|v| v.as_str()).unwrap_or("");
if ty != "playlist" {
bail!("yt-dlp output _type is {ty:?}, expected \"playlist\"");
}
let playlist_id = json
.get("id")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
let raw_entries = json
.get("entries")
.and_then(|v| v.as_array())
.map(|a| a.as_slice())
.unwrap_or(&[]);
// Infer the fallback base host from the container URL so YouTube Music
// entries stay on music.youtube.com rather than www.youtube.com.
let fallback_host = if url.contains("music.youtube.com") {
"music.youtube.com"
} else {
"www.youtube.com"
};
let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://");
let mut items = Vec::with_capacity(raw_entries.len());
for entry in raw_entries {
if entry.is_null() {
continue; // unavailable/private video in flat listing
}
let id = match entry.get("id").and_then(|v| v.as_str()) {
Some(s) => s.to_owned(),
None => continue,
};
// Normalize the item URL:
// 1. Prefer `webpage_url` — yt-dlp always makes this absolute when present.
// 2. Accept `url` only when it is already an absolute HTTP(S) URL; in
// flat-playlist mode `url` is often just the bare video ID.
// 3. Fallback: construct from `id` using the same host as the container URL.
let item_url = entry
.get("webpage_url")
.and_then(|v| v.as_str())
.filter(|s| is_absolute(s))
.map(str::to_owned)
.or_else(|| {
entry
.get("url")
.and_then(|v| v.as_str())
.filter(|s| is_absolute(s))
.map(str::to_owned)
})
.unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}"));
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let item_uploader = entry.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
items.push(PlaylistItem { id, url: item_url, title: item_title, uploader: item_uploader });
}
Ok(PlaylistInfo { playlist_id, title, uploader, items })
}
#[cfg(test)]
mod tests {
use super::{available_video_heights, has_audio_track, quality_format};

220
docs/superpowers/plans/2026-07-20-playlist-children.md Normal file
View file

@ -0,0 +1,220 @@
# Playlist / Channel Captures + Parent–Child Entry Display
**Date:** 2026-07-20
**Branch:** `feat-captures-with-children`
---
## Goal
1. **YouTube (Music) playlists and channels** — when the user submits a playlist or channel URL/shorthand, archive every video in it as a tree of entries: one root "container" entry for the playlist/channel, one child entry per video.
2. **Parent–child display** — the entry list and detail view can show children of a container entry in a collapsible inline section, replacing the current flat/single-entry-only model.
---
## Schema Contract (no migrations needed — columns already exist)
| Column | Meaning |
|---|---|
| `archived_entries.parent_entry_id` | `NULL` for root entries; points to container id for children |
| `archived_entries.root_entry_id` | Same as `parent_entry_id` for depth-1 children (direct children of a container) |
| `archive_run_items.parent_item_id` | `NULL` for the container item; points to container item id for video items |
| `archive_runs.requested_count` | Always `1` (user submitted one locator) |
| `archive_runs.discovered_count` | `N + 1` — container item + N video items (from `refresh_run_counters`) |
| `archive_runs.completed_count` | Items completed (container + each successful video) |
| `archive_runs.failed_count` | Items that failed to download |
**Counter note:** `refresh_run_counters` counts every `archive_run_items` row, so `discovered_count = 1 + N` when a playlist has N videos. This is intentional and accurate — the container item itself counts as a discovered/completed item.
---
## yt-dlp Probe Strategy
For container sources (playlist, channel, YouTube Music playlist), use:
```bash
yt-dlp -J --flat-playlist <url>
```
`-J` (`--dump-single-json`) returns a **single JSON object** for the whole container — reliable `title` and `uploader` fields at the top level, plus an `entries` array (shallow per-item objects with at minimum `id`, `url`, `title`). This avoids the per-item `playlist_title` reliability issue that `--dump-json` (multi-line) has.
Individual child video downloads stay single-item (`--no-playlist`), same as current video capture.
---
## File Map
| File | Action | What changes |
|---|---|---|
| `crates/archivr-core/src/downloader/ytdlp.rs` | Modify | Add `PlaylistItem`, `PlaylistInfo` structs; `fetch_playlist_info()` |
| `crates/archivr-core/src/capture.rs` | Modify | `record_media_entry` gains `parent_entry_id`/`root_entry_id` params; add `record_container_entry()`; implement playlist/channel/YTM-playlist capture paths |
| `crates/archivr-core/src/archive.rs` | Modify | Add `child_count: i64` to `EntrySummary`; add `get_entry_summary()`; fix `get_entry_detail()` to handle child entries; add `list_child_entries()` |
| `crates/archivr-server/src/routes.rs` | Modify | Add `GET /api/archives/:id/entries/:uid/children` route + handler |
| `frontend/src/api.js` | Modify | Add `fetchEntryChildren()` |
| `frontend/src/components/EntryRow.jsx` | Modify | Expand toggle when `child_count > 0`; inline child sub-rows on expand |
| `frontend/src/styles.css` | Modify | Child row indentation + expand toggle styling |
---
## Detailed Design
### `ytdlp.rs` additions
```rust
pub struct PlaylistItem {
pub id: String,
pub url: String,
pub title: Option<String>,
pub uploader: Option<String>,
}
pub struct PlaylistInfo {
pub playlist_id: String,
pub title: Option<String>,
pub uploader: Option<String>,
pub items: Vec<PlaylistItem>,
}
/// Runs `yt-dlp -J --flat-playlist <url>` and parses the result.
/// Returns Err if yt-dlp fails or the output isn't a playlist object.
pub fn fetch_playlist_info(url: &str, cookies: &HashMap<String, String>) -> Result<PlaylistInfo>
```
### `capture.rs` changes
**`record_media_entry` signature extension:**
```rust
fn record_media_entry(
...,
title: Option<String>,
parent_entry_id: Option<i64>, // NEW
root_entry_id: Option<i64>, // NEW
) -> Result<database::ArchivedEntry>
```
All existing call sites pass `None, None`. Playlist child calls pass the container entry's ids.
**New `record_container_entry()`:**
- Creates an entry with no blob and no `primary_media` artifact.
- `source_kind/entity_kind` from `source_metadata(source)`.
- `parent_entry_id: None, root_entry_id: None`.
- Stores playlist metadata in `source_metadata_json`.
- Returns the `ArchivedEntry` (needed for child `parent_entry_id`).
- Calls `database::complete_archive_run_item()` on the container run item.
**Playlist/channel capture path (replaces `return Err(...)` stubs):**
```
1. fetch_playlist_info(url, cookies) → PlaylistInfo
2. create_archive_run(conn, user_id, 1) → run (requested_count=1)
3. create_archive_run_item(run, None, 0, ..., "playlist"/"channel", "container")
4. record_container_entry(...) → container_entry
5. complete_archive_run_item(container_item, container_entry.id)
6. for (ordinal, item) in playlist_info.items:
a. create_archive_run_item(run, Some(container_item.id), ordinal, item.url, ...)
b. fetch_metadata(item.url, cookies) → metadata_json (for title)
c. ytdlp::download(item.url, ...) → (hash, ext)
d. record_media_entry(..., Some(container.id), Some(container.id))
OR: fail_archive_run_item(child_item, error) and continue
7. finish_archive_run(conn, run.id)
```
Error handling: if a child video fails, call `fail_archive_run_item` and continue — partial success is correct for playlists.
### `archive.rs` changes
**`EntrySummary` new field:**
```rust
pub child_count: i64, // number of direct children; 0 for non-container entries
```
**`ENTRY_SELECT_COLS` extension** (adds col 12):
```sql
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count
```
**`list_root_entries` inline SQL** also extended the same way (col 12).
All `query_map` closures that build `EntrySummary` get `child_count: row.get(12)?`.
**`get_entry_summary(conn, entry_uid)`** — new private helper:
- Fetches one entry by uid without the `parent_entry_id IS NULL` constraint.
- Used by the fixed `get_entry_detail`.
**`get_entry_detail` fix:**
```rust
// BEFORE (broken for child entries):
let summary = list_root_entries(conn, u32::MAX)?
.into_iter()
.find(|entry| entry.entry_uid == entry_uid)
.context("entry disappeared")?;
// AFTER:
let summary = get_entry_summary(conn, entry_uid)?
.context("entry disappeared")?;
```
**New `list_child_entries(conn, parent_uid) -> Result<Vec<EntrySummary>>`:**
```sql
ENTRY_SELECT_COLS ENTRY_FROM_JOINS
WHERE e.parent_entry_id = (SELECT id FROM archived_entries WHERE entry_uid = ?1)
GROUP BY e.id
ORDER BY e.archived_at ASC, e.id ASC
```
(ascending order — preserves playlist ordinal feel)
### `routes.rs` addition
```
.route(
"/api/archives/:archive_id/entries/:entry_uid/children",
get(list_entry_children),
)
```
```rust
async fn list_entry_children(
State(state): State<AppState>,
auth: AuthUser,
Path((archive_id, entry_uid)): Path<(String, String)>,
) -> Result<Json<Vec<archive::EntrySummary>>, ApiError> {
auth.require_auth()?;
let mounted = mounted_archive(&state, &archive_id)?;
let conn = database::open_or_initialize(&mounted.archive_path)?;
Ok(Json(archive::list_child_entries(&conn, &entry_uid)?))
}
```
### Frontend changes
**`api.js`:**
```js
export async function fetchEntryChildren(archiveId, entryUid) {
return getJson(`/api/archives/${archiveId}/entries/${entryUid}/children`);
}
```
**`EntryRow.jsx`:**
- Accept optional `archiveId` for child fetching (already passed).
- When `entry.child_count > 0`: render a chevron expand button in `col-title` area.
- Local state: `expanded` (bool), `children` (array | null), `loading` (bool).
- On chevron click: toggle; if expanding and `children === null`, call `fetchEntryChildren` and store result.
- Render children as `<div className="child-entries">` containing simplified `<ChildEntryRow>` elements (or reuse `EntryRow` without nesting).
**`styles.css`:**
- `.child-entries` — slight left indent, separator line.
- `.entry-expand-btn` — minimal chevron button.
---
## Acceptance Criteria
- `cargo test -p archivr-core` green.
- `cargo check -p archivr-server` clean.
- Submitting `yt:playlist/PLxxx` archives the playlist as a container entry with N child video entries in the DB.
- Submitting `yt:@handle` or `yt:channel/UC...` does the same for channel uploads.
- Submitting `ytm:playlist/PLxxx` does the same for YouTube Music playlists.
- Individual video/track captures (`yt:video/ID`, `ytm:ID`) unchanged.
- If one video in a playlist fails, the run status is `failed` but other children are still archived.
- `GET /api/archives/:id/entries` returns root entries only, each with correct `child_count`.
- `GET /api/archives/:id/entries/:uid` works for both root and child entries.
- `GET /api/archives/:id/entries/:uid/children` returns the child entries for a container.
- Frontend expand button appears on container entries; clicking it fetches and shows children inline.