mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
feat(core): playlist per-item quality + incremental sync
ytdlp.rs:
- Add PlaylistItemProbe / PlaylistProbeResult (pub, serde::Serialize)
- Add private available_video_heights_from_value() helper for Value entries
- Add probe_playlist_qualities(): yt-dlp -J (full metadata, no flat flag)
returns per-video quality lists in one subprocess call
capture.rs:
- Add per_item_quality: HashMap<String,String> and sync: bool to CaptureConfig
(both Default; keyed by yt-dlp video ID, not URL)
- Add pub locator_to_playlist_url(): validates only the three playlist sources,
expands shorthands; keeps locator_to_ytdlp_url's no-playlists contract
- Playlist capture block: sync-aware container resolution
- sync + existing container → reuse it via complete_archive_run_item,
skip already-archived children (by canonical URL) before creating
run items so refresh_run_counters only counts new items
- sync + no container → create normally (first sync run)
- non-sync → always create fresh container (existing behaviour)
- Per-item quality: config.per_item_quality.get(id) falls back to child_quality
archive.rs:
- Add get_archived_playlist_child_urls(): returns HashSet of canonical URLs
of all children under any container matching the playlist canonical URL
- Add find_container_entry_id_by_canonical_url(): returns most-recent
container entry id (parent_entry_id IS NULL) for a given canonical URL
routes.rs: stub per_item_quality/sync on both CaptureConfig sites (server
agent will wire body fields in Phase 2)
This commit is contained in:
parent
9c1d416463
commit
814aa76a1d
4 changed files with 277 additions and 26 deletions
|
|
@ -7,7 +7,7 @@ use std::{
|
|||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use crate::{archive::ArchivePaths, database, downloader, twitter::parse_tweet_id};
|
||||
use crate::{archive::{self, ArchivePaths}, database, downloader, twitter::parse_tweet_id};
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
|
||||
pub enum Source {
|
||||
|
|
@ -90,6 +90,14 @@ pub struct CaptureConfig {
|
|||
/// Route WebPage captures through the Freedium mirror to bypass paywalls.
|
||||
/// The original locator is still recorded in the DB; only the fetch URL changes.
|
||||
pub via_freedium: bool,
|
||||
/// Per-item quality overrides for playlist captures.
|
||||
/// Keys are yt-dlp video IDs (e.g. "dQw4w9WgXcQ"); values are quality strings
|
||||
/// accepted by `quality_format` ("best", "audio", "1080p", etc.).
|
||||
/// Empty map = apply the global `quality` to all items.
|
||||
pub per_item_quality: HashMap<String, String>,
|
||||
/// When true, skip playlist items whose URL is already archived as a child
|
||||
/// of any container entry with the same canonical playlist URL.
|
||||
pub sync: bool,
|
||||
}
|
||||
|
||||
/// Resolves which cookies apply to `url` by evaluating all rules in ordinal order.
|
||||
|
|
@ -536,6 +544,19 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Returns the canonical URL for playlist/channel locators that yt-dlp can
|
||||
/// expand with `--flat-playlist`, or `None` for non-playlist sources.
|
||||
/// Intentionally separate from `locator_to_ytdlp_url`, which excludes playlists.
|
||||
pub fn locator_to_playlist_url(locator: &str) -> Option<String> {
|
||||
let source = determine_source(locator);
|
||||
match source {
|
||||
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist => {
|
||||
Some(expand_shorthand_to_url(locator, &source))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result<bool> {
|
||||
let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?);
|
||||
Ok(path.exists())
|
||||
|
|
@ -1049,28 +1070,58 @@ pub fn perform_capture(
|
|||
.clone()
|
||||
.or_else(|| playlist_info.uploader.clone().map(|u| format!("{u} (channel)")));
|
||||
|
||||
let container_entry = match record_container_entry(
|
||||
&conn,
|
||||
store_path,
|
||||
user_id,
|
||||
&run,
|
||||
&item,
|
||||
locator,
|
||||
&canonical_url,
|
||||
source,
|
||||
container_title,
|
||||
&playlist_info.playlist_id,
|
||||
playlist_info.uploader.as_deref(),
|
||||
) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
return Err(fail_run(
|
||||
&conn,
|
||||
&run,
|
||||
&item,
|
||||
&format!("Failed to create container entry: {e:#}"),
|
||||
));
|
||||
// Sync mode: reuse an existing container so we don't create a duplicate
|
||||
// root entry on every sync run. Non-sync always creates a fresh container.
|
||||
let (container_id, already_archived) = if config.sync {
|
||||
match archive::find_container_entry_id_by_canonical_url(&conn, &canonical_url) {
|
||||
Err(e) => {
|
||||
return Err(fail_run(
|
||||
&conn, &run, &item,
|
||||
&format!("Failed to query existing container: {e:#}"),
|
||||
));
|
||||
}
|
||||
Ok(Some(existing_id)) => {
|
||||
// Container already exists — mark the run item done pointing at it;
|
||||
// don't call record_container_entry (that would create a duplicate).
|
||||
if let Err(e) = database::complete_archive_run_item(&conn, item.id, existing_id) {
|
||||
return Err(fail_run(
|
||||
&conn, &run, &item,
|
||||
&format!("Failed to complete run item for existing container: {e:#}"),
|
||||
));
|
||||
}
|
||||
let archived = archive::get_archived_playlist_child_urls(&conn, &canonical_url)
|
||||
.unwrap_or_default();
|
||||
(existing_id, archived)
|
||||
}
|
||||
Ok(None) => {
|
||||
// First sync run for this playlist — create the container normally.
|
||||
let e = match record_container_entry(
|
||||
&conn, store_path, user_id, &run, &item, locator, &canonical_url,
|
||||
source, container_title, &playlist_info.playlist_id,
|
||||
playlist_info.uploader.as_deref(),
|
||||
) {
|
||||
Ok(e) => e,
|
||||
Err(e) => return Err(fail_run(
|
||||
&conn, &run, &item,
|
||||
&format!("Failed to create container entry: {e:#}"),
|
||||
)),
|
||||
};
|
||||
(e.id, std::collections::HashSet::new())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let e = match record_container_entry(
|
||||
&conn, store_path, user_id, &run, &item, locator, &canonical_url,
|
||||
source, container_title, &playlist_info.playlist_id,
|
||||
playlist_info.uploader.as_deref(),
|
||||
) {
|
||||
Ok(e) => e,
|
||||
Err(e) => return Err(fail_run(
|
||||
&conn, &run, &item,
|
||||
&format!("Failed to create container entry: {e:#}"),
|
||||
)),
|
||||
};
|
||||
(e.id, std::collections::HashSet::new())
|
||||
};
|
||||
|
||||
let is_audio = source == Source::YouTubeMusicPlaylist;
|
||||
|
|
@ -1078,6 +1129,12 @@ pub fn perform_capture(
|
|||
let child_entity_kind = if is_audio { "music" } else { "video" };
|
||||
|
||||
for (ordinal, playlist_item) in playlist_info.items.iter().enumerate() {
|
||||
// Sync: skip items already archived — no run item created, so
|
||||
// refresh_run_counters sees only newly-attempted items.
|
||||
if config.sync && already_archived.contains(&playlist_item.url) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let child_timestamp = format!(
|
||||
"{}-{}",
|
||||
Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
|
||||
|
|
@ -1114,12 +1171,19 @@ pub fn perform_capture(
|
|||
None => playlist_item.title.clone(),
|
||||
};
|
||||
|
||||
// Per-item quality override keyed by yt-dlp video ID; fall back to playlist-level.
|
||||
let effective_child_quality: Option<&str> = config
|
||||
.per_item_quality
|
||||
.get(&playlist_item.id)
|
||||
.map(String::as_str)
|
||||
.or(child_quality);
|
||||
|
||||
// Download the media.
|
||||
match downloader::ytdlp::download(
|
||||
playlist_item.url.clone(),
|
||||
store_path,
|
||||
&child_timestamp,
|
||||
child_quality,
|
||||
effective_child_quality,
|
||||
&cookies,
|
||||
) {
|
||||
Ok((hash, file_extension)) => {
|
||||
|
|
@ -1165,8 +1229,8 @@ pub fn perform_capture(
|
|||
&file_extension,
|
||||
byte_size,
|
||||
child_title,
|
||||
Some(container_entry.id),
|
||||
Some(container_entry.id),
|
||||
Some(container_id),
|
||||
Some(container_id),
|
||||
) {
|
||||
Ok(child_entry) => {
|
||||
let _ = database::refresh_entry_cached_bytes(&conn, child_entry.id);
|
||||
|
|
@ -1191,7 +1255,7 @@ pub fn perform_capture(
|
|||
}
|
||||
}
|
||||
|
||||
database::refresh_entry_cached_bytes(&conn, container_entry.id)?;
|
||||
database::refresh_entry_cached_bytes(&conn, container_id)?;
|
||||
database::finish_archive_run(&conn, run.id)?;
|
||||
|
||||
let run_status: String = conn
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue