thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 03:05:32 +02:00
Code Issues Activity

feat(core): YouTube playlist/channel/YTM-playlist capture with parent–child entries

Browse source 
- ytdlp: add fetch_playlist_info() using yt-dlp -J --flat-playlist for
  reliable container title + shallow entry list; normalize item URLs via
  webpage_url → absolute url → id fallback (domain inferred from container
  URL so YTM stays on music.youtube.com)
- capture: add record_container_entry() (no blob, no primary_media artifact);
  extend record_media_entry() with parent_entry_id/root_entry_id params (all
  existing single-item call sites pass None, None)
- capture: implement YouTubePlaylist / YouTubeChannel / YouTubeMusicPlaylist
  capture path replacing the two not-implemented stubs: fetch playlist info →
  create container entry (reusing existing run + item) → per-child run items
  (parent_item_id = container item) → download each video/track as a child
  entry; per-child failures are non-fatal; perform_capture returns result.status
  reflecting actual run outcome so capture_handler marks the job correctly
- archive: add child_count i64 to EntrySummary (col 12 in all listing queries);
  add get_entry_summary() private helper; fix get_entry_detail() to use
  get_entry_summary() so child entries are resolvable via the detail endpoint;
  add list_child_entries(conn, uid, caller_bits) with the same
  admin/collection visibility predicate as list_root_entries

archive_runs.requested_count stays 1 (one user locator); discovered/
completed/failed_count reflect container item + N video items via
refresh_run_counters.
This commit is contained in:
TheGeneralist 2026-07-20 16:07:00 +02:00
parent d202e177e1
commit ccdacfd582
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
5 changed files with 689 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

101
crates/archivr-core/src/archive.rs
View file

@ -30,6 +30,8 @@ pub struct EntrySummary {
pub has_favicon: bool,
/// Bytes of blobs already on disk from an earlier entry (precomputed at capture time).
pub cached_bytes: i64,
/// Number of direct child entries; 0 for non-container entries.
pub child_count: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
@ -215,7 +217,8 @@ pub fn list_root_entries(
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
NULL AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon,
e.cached_bytes
e.cached_bytes,
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count
FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -248,6 +251,7 @@ pub fn list_root_entries(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -255,6 +259,39 @@ pub fn list_root_entries(
Ok(entries)
}
/// Fetches one `EntrySummary` for any entry (root or child) by uid.
/// Returns `None` if not found.
fn get_entry_summary(
conn: &rusqlite::Connection,
entry_uid: &str,
) -> Result<Option<EntrySummary>> {
let sql = format!(
"{} {} WHERE e.entry_uid = ?1 GROUP BY e.id",
ENTRY_SELECT_COLS, ENTRY_FROM_JOINS,
);
let mut stmt = conn.prepare(&sql)?;
let result = stmt
.query_row([entry_uid], |row| {
Ok(EntrySummary {
entry_uid: row.get(0)?,
archived_at: row.get(1)?,
source_kind: row.get(2)?,
entity_kind: row.get(3)?,
title: row.get(4)?,
visibility: row.get(5)?,
original_url: row.get(6)?,
artifact_count: row.get(7)?,
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})
.optional()?;
Ok(result)
}
pub fn get_entry_detail(
conn: &rusqlite::Connection,
entry_uid: &str,
@ -279,9 +316,7 @@ pub fn get_entry_detail(
return Ok(None);
};
let summary = list_root_entries(conn, u32::MAX)?
.into_iter()
.find(|entry| entry.entry_uid == entry_uid)
let summary = get_entry_summary(conn, entry_uid)?
.context("entry disappeared while loading detail")?;
let mut stmt = conn.prepare(
@ -438,12 +473,62 @@ pub fn list_entries_for_collection(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(entries)
}
/// Returns the direct children of the entry identified by `parent_entry_uid`,
/// ordered ascending by `archived_at, id` (preserves playlist ordinal feel).
/// Returns an empty vec if the parent has no children or does not exist.
pub fn list_child_entries(
conn: &rusqlite::Connection,
parent_entry_uid: &str,
caller_bits: u32,
) -> Result<Vec<EntrySummary>> {
let sql = format!(
"{} {} \
WHERE e.parent_entry_id = (SELECT id FROM archived_entries WHERE entry_uid = ?1) \
AND (\
CAST(?2 AS INTEGER) & 12 != 0 \
OR EXISTS (\
SELECT 1 FROM collection_entries ce \
WHERE ce.entry_id = e.id \
AND ce.visibility_bits & CAST(?2 AS INTEGER) != 0\
)\
) \
GROUP BY e.id \
ORDER BY e.archived_at ASC, e.id ASC",
ENTRY_SELECT_COLS, ENTRY_FROM_JOINS,
);
let mut stmt = conn.prepare(&sql)?;
let entries = stmt
.query_map(
rusqlite::params![parent_entry_uid, caller_bits as i64],
|row| {
Ok(EntrySummary {
entry_uid: row.get(0)?,
archived_at: row.get(1)?,
source_kind: row.get(2)?,
entity_kind: row.get(3)?,
title: row.get(4)?,
visibility: row.get(5)?,
original_url: row.get(6)?,
artifact_count: row.get(7)?,
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
},
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(entries)
}
/// Resolves an artifact to its absolute on-disk path under `store_path`.
///
/// `artifact.relpath` is a store-relative path (e.g. `raw/a/b/abc.pdf`).
@ -555,7 +640,8 @@ const ENTRY_SELECT_COLS: &str = "SELECT e.entry_uid, e.archived_at, e.source_kin
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \
parent.entry_uid AS parent_entry_uid, \
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, \
e.cached_bytes";
e.cached_bytes, \
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count";
const ENTRY_FROM_JOINS: &str = "FROM archived_entries e \
JOIN source_identities si ON si.id = e.source_identity_id \
@ -663,6 +749,7 @@ pub fn search_entries(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -870,7 +957,8 @@ pub fn entries_for_tag(
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
parent.entry_uid AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon,
e.cached_bytes
e.cached_bytes,
(SELECT COUNT(*) FROM archived_entries child WHERE child.parent_entry_id = e.id) AS child_count
FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -896,6 +984,7 @@ pub fn entries_for_tag(
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
child_count: row.get(12)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;

265
crates/archivr-core/src/capture.rs
View file

@ -660,6 +660,8 @@ fn record_media_entry(
file_extension: &str,
byte_size: i64,
title: Option<String>,
parent_entry_id: Option<i64>,
root_entry_id: Option<i64>,
) -> Result<database::ArchivedEntry> {
debug_assert!(run.run_uid.starts_with("run_"));
debug_assert!(item.item_uid.starts_with("item_"));
@ -686,8 +688,8 @@ fn record_media_entry(
&database::NewEntry {
source_identity_id,
archive_run_id: run.id,
parent_entry_id: None,
root_entry_id: None,
parent_entry_id,
root_entry_id,
created_by_user_id: user_id,
owned_by_user_id: user_id,
source_kind: source_kind.to_string(),
@ -720,6 +722,59 @@ fn record_media_entry(
Ok(entry)
}
/// Creates a playlist/channel container entry with no blob and no primary-media artifact.
/// Calls `complete_archive_run_item` on the provided item.
fn record_container_entry(
conn: &rusqlite::Connection,
store_path: &Path,
user_id: i64,
run: &database::ArchiveRun,
item: &database::ArchiveRunItem,
requested_locator: &str,
canonical_locator: &str,
source: Source,
title: Option<String>,
playlist_id: &str,
uploader: Option<&str>,
) -> Result<database::ArchivedEntry> {
let (source_kind, entity_kind, representation_kind) = source_metadata(source);
let source_identity_id = database::upsert_source_identity(
conn,
source_kind,
entity_kind,
Some(playlist_id),
Some(canonical_locator),
canonical_locator,
)?;
let entry = database::create_archived_entry(
conn,
&database::NewEntry {
source_identity_id,
archive_run_id: run.id,
parent_entry_id: None,
root_entry_id: None,
created_by_user_id: user_id,
owned_by_user_id: user_id,
source_kind: source_kind.to_string(),
entity_kind: entity_kind.to_string(),
title,
visibility: "private".to_string(),
representation_kind: representation_kind.to_string(),
source_metadata_json: json!({
"requested_locator": requested_locator,
"canonical_locator": canonical_locator,
"playlist_id": playlist_id,
"uploader": uploader,
})
.to_string(),
display_metadata_json: None,
},
)?;
create_structured_root(store_path, &entry)?;
database::complete_archive_run_item(conn, item.id, entry.id)?;
Ok(entry)
}
/// Extracts PlatformMetadata from a tweet JSON string.
/// Returns Default on any parse failure.
fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
@ -968,14 +1023,191 @@ pub fn perform_capture(
));
}
// Sources: YouTube Music Playlist — container expansion not yet implemented.
if source == Source::YouTubeMusicPlaylist {
return Err(fail_run(
// Sources: YouTube playlists, YouTube channels, YouTube Music playlists.
// Fetch container metadata, create a root container entry, then download
// each video/track as a child entry.
// `run` and `item` are already created above; `item` acts as the container item.
if matches!(
source,
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist
) {
// `canonical_url` already holds the expanded URL (same value as `path` later).
let playlist_info = match downloader::ytdlp::fetch_playlist_info(&canonical_url, &cookies) {
Ok(info) => info,
Err(e) => {
return Err(fail_run(
&conn,
&run,
&item,
&format!("Failed to fetch playlist info: {e:#}"),
));
}
};
let container_title = playlist_info
.title
.clone()
.or_else(|| playlist_info.uploader.clone().map(|u| format!("{u} (channel)")));
let container_entry = match record_container_entry(
&conn,
store_path,
user_id,
&run,
&item,
"YouTube Music playlist archiving is not yet implemented.",
));
locator,
&canonical_url,
source,
container_title,
&playlist_info.playlist_id,
playlist_info.uploader.as_deref(),
) {
Ok(e) => e,
Err(e) => {
return Err(fail_run(
&conn,
&run,
&item,
&format!("Failed to create container entry: {e:#}"),
));
}
};
let is_audio = source == Source::YouTubeMusicPlaylist;
let child_quality: Option<&str> = if is_audio { Some("audio") } else { quality };
let child_entity_kind = if is_audio { "music" } else { "video" };
for (ordinal, playlist_item) in playlist_info.items.iter().enumerate() {
let child_timestamp = format!(
"{}-{}",
Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
Uuid::new_v4().simple(),
);
let child_item = match database::create_archive_run_item(
&conn,
run.id,
Some(item.id),
ordinal as i64,
&playlist_item.url,
Some(&playlist_item.url),
source_kind,
child_entity_kind,
) {
Ok(i) => i,
Err(e) => {
eprintln!("warn: playlist item {} create_run_item failed: {e:#}", playlist_item.url);
continue;
}
};
// Fetch metadata for the child title (best-effort).
let child_meta_json = downloader::ytdlp::fetch_metadata(&playlist_item.url, &cookies);
let child_title: Option<String> = match &child_meta_json {
Some(json) => {
let meta = downloader::metadata::extract_from_ytdlp_json(json);
Some(generate_entry_title(
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo },
&meta,
))
}
None => playlist_item.title.clone(),
};
// Download the media.
match downloader::ytdlp::download(
playlist_item.url.clone(),
store_path,
&child_timestamp,
child_quality,
&cookies,
) {
Ok((hash, file_extension)) => {
let temp_file = store_path
.join("temp")
.join(&child_timestamp)
.join(format!("{child_timestamp}{file_extension}"));
let byte_size = match fs::metadata(&temp_file) {
Ok(m) => m.len() as i64,
Err(e) => {
eprintln!("warn: stat child temp file: {e:#}");
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("failed to stat downloaded file: {e:#}"),
);
continue;
}
};
if !hash_exists(&hash, &file_extension, store_path).unwrap_or(false) {
if let Err(e) = move_temp_to_raw(&temp_file, &hash, store_path) {
eprintln!("warn: move_temp_to_raw child: {e:#}");
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("failed to move downloaded file: {e:#}"),
);
continue;
}
}
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
let child_source =
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo };
match record_media_entry(
&conn,
store_path,
user_id,
&run,
&child_item,
&playlist_item.url,
&playlist_item.url,
child_source,
&hash,
&file_extension,
byte_size,
child_title,
Some(container_entry.id),
Some(container_entry.id),
) {
Ok(child_entry) => {
let _ = database::refresh_entry_cached_bytes(&conn, child_entry.id);
}
Err(e) => {
eprintln!("warn: record child entry: {e:#}");
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("failed to record entry: {e:#}"),
);
}
}
}
Err(e) => {
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
eprintln!("warn: yt-dlp child download failed for {}: {e:#}", playlist_item.url);
let _ = database::fail_archive_run_item(
&conn, child_item.id,
&format!("yt-dlp download failed: {e:#}"),
);
}
}
}
database::refresh_entry_cached_bytes(&conn, container_entry.id)?;
database::finish_archive_run(&conn, run.id)?;
let run_status: String = conn
.query_row(
"SELECT status FROM archive_runs WHERE id = ?1",
[run.id],
|row| row.get(0),
)
.unwrap_or_else(|_| "completed".to_string());
return Ok(CaptureResult {
run_uid: run.run_uid.clone(),
status: run_status,
ublock_skipped: false,
cookie_ext_skipped: false,
});
}
// Source: generic HTTP/S file URL
@ -1018,6 +1250,8 @@ pub fn perform_capture(
&file_extension,
byte_size,
title_hint,
None,
None,
)?;
database::refresh_entry_cached_bytes(&conn, entry.id)?;
database::finish_archive_run(&conn, run.id)?;
@ -1144,8 +1378,10 @@ pub fn perform_capture(
&html_hash,
&file_extension,
byte_size,
entry_title,
)?;
entry_title,
None,
None,
)?;
// 5. Add favicon artifact if we captured one.
if let Some((fav_relpath, fav_blob_id)) = favicon_info {
@ -1342,14 +1578,7 @@ pub fn perform_capture(
}
}
}
Source::YouTubePlaylist | Source::YouTubeChannel => {
return Err(fail_run(
&conn,
&run,
&item,
"Playlist and channel container expansion are not yet implemented.",
));
}
Source::YouTubePlaylist | Source::YouTubeChannel => unreachable!(),
_ => unreachable!(),
};
let temp_file = store_path
@ -1402,6 +1631,8 @@ pub fn perform_capture(
&file_extension,
byte_size,
entry_title,
None,
None,
)?;
database::refresh_entry_cached_bytes(&conn, media_entry.id)?;
database::finish_archive_run(&conn, run.id)?;

124
crates/archivr-core/src/downloader/ytdlp.rs
View file

@ -6,10 +6,29 @@ use std::{
process::Command,
};
use uuid::Uuid;
use serde_json;
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
use crate::hash::hash_file;
/// A single item in a flat playlist listing from `fetch_playlist_info`.
#[derive(Debug)]
pub struct PlaylistItem {
pub id: String,
pub url: String,
pub title: Option<String>,
pub uploader: Option<String>,
}
/// Container metadata returned by `fetch_playlist_info`.
#[derive(Debug)]
pub struct PlaylistInfo {
pub playlist_id: String,
pub title: Option<String>,
pub uploader: Option<String>,
pub items: Vec<PlaylistItem>,
}
/// Returns the yt-dlp `-f` format selector for `quality`.
///
/// - `"audio"` → prefers native Opus/WebM (most efficient), then native
@ -241,6 +260,111 @@ pub fn fetch_metadata(path: &str, cookies: &HashMap<String, String>) -> Option<S
if json.trim().is_empty() { None } else { Some(json) }
}
/// Runs `yt-dlp -J --flat-playlist <url>` and parses the single-JSON result.
///
/// `-J` / `--dump-single-json` returns one JSON object for the whole
/// container with reliable top-level `title` / `uploader` fields plus an
/// `entries` array of shallow per-item objects.
///
/// Returns an error if yt-dlp fails, the output is not valid JSON, or
/// the root `_type` is not `"playlist"`.
pub fn fetch_playlist_info(url: &str, cookies: &HashMap<String, String>) -> Result<PlaylistInfo> {
let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
let domain = domain_from_url(url);
let p = std::env::temp_dir()
.join(format!("archivr-cookies-{}.txt", Uuid::new_v4().simple()));
write_netscape_cookie_file(cookies, &domain, &p)
.context("failed to write yt-dlp cookie file")?;
Some(p)
} else {
None
};
let mut cmd = std::process::Command::new(&ytdlp);
cmd.arg("-J").arg("--flat-playlist");
if let Some(cf) = &cookie_file {
cmd.arg("--cookies").arg(cf);
}
cmd.arg(url);
let out = cmd.output();
if let Some(cf) = &cookie_file {
let _ = std::fs::remove_file(cf);
}
let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?;
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr);
bail!("yt-dlp -J --flat-playlist failed for {url}: {stderr}");
}
let json: serde_json::Value = serde_json::from_slice(&out.stdout)
.context("yt-dlp -J output is not valid JSON")?;
let ty = json.get("_type").and_then(|v| v.as_str()).unwrap_or("");
if ty != "playlist" {
bail!("yt-dlp output _type is {ty:?}, expected \"playlist\"");
}
let playlist_id = json
.get("id")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
let raw_entries = json
.get("entries")
.and_then(|v| v.as_array())
.map(|a| a.as_slice())
.unwrap_or(&[]);
// Infer the fallback base host from the container URL so YouTube Music
// entries stay on music.youtube.com rather than www.youtube.com.
let fallback_host = if url.contains("music.youtube.com") {
"music.youtube.com"
} else {
"www.youtube.com"
};
let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://");
let mut items = Vec::with_capacity(raw_entries.len());
for entry in raw_entries {
if entry.is_null() {
continue; // unavailable/private video in flat listing
}
let id = match entry.get("id").and_then(|v| v.as_str()) {
Some(s) => s.to_owned(),
None => continue,
};
// Normalize the item URL:
// 1. Prefer `webpage_url` — yt-dlp always makes this absolute when present.
// 2. Accept `url` only when it is already an absolute HTTP(S) URL; in
// flat-playlist mode `url` is often just the bare video ID.
// 3. Fallback: construct from `id` using the same host as the container URL.
let item_url = entry
.get("webpage_url")
.and_then(|v| v.as_str())
.filter(|s| is_absolute(s))
.map(str::to_owned)
.or_else(|| {
entry
.get("url")
.and_then(|v| v.as_str())
.filter(|s| is_absolute(s))
.map(str::to_owned)
})
.unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}"));
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let item_uploader = entry.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
items.push(PlaylistItem { id, url: item_url, title: item_title, uploader: item_uploader });
}
Ok(PlaylistInfo { playlist_id, title, uploader, items })
}
#[cfg(test)]
mod tests {
use super::{available_video_heights, has_audio_track, quality_format};