From 4e7bd086ea9c28f15e5664d1a6ace407bc6a0afc Mon Sep 17 00:00:00 2001 From: archivr-qa Date: Sun, 23 Aug 2026 19:20:58 +0200 Subject: [PATCH 1/2] feat(core): resolve_yt_dlp picks the newer of pinned vs state-dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nix flake wrapper pins a yt-dlp via ARCHIVR_YT_DLP, but yt-dlp rots fast — extractors break within weeks of a pin. Add a resolver that probes `--version` on both the pinned binary and a user-installed copy under the mutable state dir, and runs whichever is newer. Version strings are YYYY.MM.DD, so plain string ordering is chronological. Ties resolve toward the state dir: a user who installed it there did so deliberately. ARCHIVR_YT_DLP_FORCE bypasses the comparison entirely, and with no candidate at all we fall back to bare `yt-dlp` on PATH — exactly the previous behaviour. Resolution is cached in a OnceLock so `--version` costs one subprocess per process, and all four inline env::var lookups now go through it. Co-Authored-By: Claude Opus 5 --- crates/archivr-core/src/downloader/ytdlp.rs | 244 +++++++++++++++++++- 1 file changed, 236 insertions(+), 8 deletions(-) diff --git a/crates/archivr-core/src/downloader/ytdlp.rs b/crates/archivr-core/src/downloader/ytdlp.rs index 26c091d..e55943d 100644 --- a/crates/archivr-core/src/downloader/ytdlp.rs +++ b/crates/archivr-core/src/downloader/ytdlp.rs @@ -4,6 +4,7 @@ use std::{ env, path::{Path, PathBuf}, process::Command, + sync::OnceLock, }; use uuid::Uuid; use serde_json; @@ -11,6 +12,109 @@ use serde_json; use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file}; use crate::hash::hash_file; +/// Env var that force-pins a specific yt-dlp binary, bypassing version comparison. +pub const YT_DLP_FORCE_ENV: &str = "ARCHIVR_YT_DLP_FORCE"; +/// Env var set by the nix flake wrapper, pointing at the pinned yt-dlp. +pub const YT_DLP_ENV: &str = "ARCHIVR_YT_DLP"; +/// Override for the mutable state directory (used by `archivr yt-dlp` and tests). +pub const STATE_DIR_ENV: &str = "ARCHIVR_STATE_DIR"; + +static RESOLVED_YT_DLP: OnceLock = OnceLock::new(); + +/// Mutable per-user state directory for archivr. +/// +/// `ARCHIVR_STATE_DIR` wins if set. Otherwise this mirrors what `dirs::state_dir()` +/// would give us without taking on the dependency: `~/Library/Application Support` +/// on macOS, `$XDG_STATE_HOME` (default `~/.local/state`) elsewhere. +pub fn state_dir() -> Option { + if let Some(dir) = env::var_os(STATE_DIR_ENV) { + if !dir.is_empty() { + return Some(PathBuf::from(dir)); + } + } + + let home = PathBuf::from(env::var_os("HOME").filter(|h| !h.is_empty())?); + + if cfg!(target_os = "macos") { + Some(home.join("Library").join("Application Support").join("archivr")) + } else { + let base = env::var_os("XDG_STATE_HOME") + .filter(|d| !d.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".local").join("state")); + Some(base.join("archivr")) + } +} + +/// Path of the user-installed (self-updated) yt-dlp inside the state dir. +pub fn state_dir_yt_dlp() -> Option { + state_dir().map(|d| d.join("yt-dlp").join("yt-dlp")) +} + +/// The nix-pinned yt-dlp advertised via `ARCHIVR_YT_DLP`, if it exists on disk. +pub fn pinned_yt_dlp() -> Option { + let p = PathBuf::from(env::var_os(YT_DLP_ENV).filter(|v| !v.is_empty())?); + p.is_file().then_some(p) +} + +/// Runs ` --version` and returns the trimmed stdout. +/// +/// yt-dlp versions are `YYYY.MM.DD`, so plain string ordering is chronological +/// ordering — no semver parsing needed. +pub fn probe_version(binary: &Path) -> Option { + let out = Command::new(binary).arg("--version").output().ok()?; + if !out.status.success() { + return None; + } + let version = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (!version.is_empty()).then_some(version) +} + +/// The candidate yt-dlp binaries, in priority order for tie-breaking +/// (later entries win ties, so the deliberately-installed state-dir copy is last). +pub fn yt_dlp_candidates() -> Vec<(&'static str, PathBuf)> { + let mut candidates = Vec::new(); + if let Some(p) = pinned_yt_dlp() { + candidates.push(("env (ARCHIVR_YT_DLP)", p)); + } + if let Some(p) = state_dir_yt_dlp() { + if p.is_file() { + candidates.push(("state-dir", p)); + } + } + candidates +} + +/// Picks the yt-dlp binary to run, without consulting the process-wide cache. +/// +/// Priority: `ARCHIVR_YT_DLP_FORCE` > newest of (pinned, state-dir) by version +/// string > bare `yt-dlp` (PATH lookup, the historical behaviour). +pub fn resolve_yt_dlp_uncached() -> PathBuf { + if let Some(forced) = env::var_os(YT_DLP_FORCE_ENV).filter(|v| !v.is_empty()) { + let forced = PathBuf::from(forced); + if forced.is_file() { + return forced; + } + } + + yt_dlp_candidates() + .into_iter() + .filter_map(|(_, path)| probe_version(&path).map(|v| (v, path))) + // `max_by` keeps the *last* maximum, and the state-dir candidate is last, + // so an exact version tie resolves in favour of the user's own install. + .max_by(|(a, _), (b, _)| a.cmp(b)) + .map(|(_, path)| path) + .unwrap_or_else(|| PathBuf::from("yt-dlp")) +} + +/// Cached [`resolve_yt_dlp_uncached`] — `--version` is spawned at most once +/// per process no matter how many yt-dlp calls the run makes. +pub fn resolve_yt_dlp() -> PathBuf { + RESOLVED_YT_DLP + .get_or_init(resolve_yt_dlp_uncached) + .clone() +} + /// A single item in a flat playlist listing from `fetch_playlist_info`. #[derive(Debug)] pub struct PlaylistItem { @@ -164,7 +268,7 @@ pub fn download( ) -> Result<(String, String)> { println!("Downloading with yt-dlp: {path}"); - let ytdlp = env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); let is_audio = quality == Some("audio"); let temp_dir = store_path.join("temp").join(timestamp); @@ -207,7 +311,7 @@ pub fn download( .arg("-o") .arg(&out_template) .output() - .with_context(|| format!("failed to spawn {ytdlp} process")); + .with_context(|| format!("failed to spawn {} process", ytdlp.display())); // Remove cookie file immediately regardless of outcome. if let Some(cf) = &cookie_file { @@ -253,7 +357,7 @@ fn find_downloaded_file(temp_dir: &Path, timestamp: &str) -> Result { /// On failure (non-zero exit or no stdout), prints the captured stderr /// to stderr (for debugging) then returns `None` so callers can proceed. pub fn fetch_metadata(path: &str, cookies: &HashMap) -> Option { - let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); // Write a temp cookie file if needed; UUID-named to avoid collisions. let cookie_file: Option = if !cookies.is_empty() { @@ -342,7 +446,7 @@ fn normalize_item_url( /// Returns an error if yt-dlp fails, the output is not valid JSON, or /// the root `_type` is not `"playlist"`. pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Result { - let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); let cookie_file: Option = if !cookies.is_empty() { let domain = domain_from_url(url); @@ -366,7 +470,7 @@ pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Resu if let Some(cf) = &cookie_file { let _ = std::fs::remove_file(cf); } - let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?; + let out = out.with_context(|| format!("failed to spawn {}", ytdlp.display()))?; if !out.status.success() { let stderr = String::from_utf8_lossy(&out.stderr); bail!("yt-dlp -J --flat-playlist failed for {url}: {stderr}"); @@ -425,7 +529,7 @@ pub fn probe_playlist_qualities( url: &str, cookies: &HashMap, ) -> Result { - let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); let cookie_file: Option = if !cookies.is_empty() { let domain = domain_from_url(url); @@ -449,7 +553,7 @@ pub fn probe_playlist_qualities( if let Some(cf) = &cookie_file { let _ = std::fs::remove_file(cf); } - let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?; + let out = out.with_context(|| format!("failed to spawn {}", ytdlp.display()))?; if !out.status.success() { let stderr = String::from_utf8_lossy(&out.stderr); bail!("yt-dlp -J failed for {url}: {stderr}"); @@ -501,7 +605,131 @@ pub fn probe_playlist_qualities( #[cfg(test)] mod tests { - use super::{available_video_heights, has_audio_track, quality_format}; + use super::{ + available_video_heights, has_audio_track, quality_format, resolve_yt_dlp_uncached, + state_dir, STATE_DIR_ENV, YT_DLP_ENV, YT_DLP_FORCE_ENV, + }; + use std::path::{Path, PathBuf}; + use std::sync::{Mutex, MutexGuard}; + + /// Env vars are process-global, so resolver tests take turns. + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + /// Clears every env var the resolver reads and hands back the serialising guard. + fn env_guard() -> MutexGuard<'static, ()> { + let guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for key in [YT_DLP_FORCE_ENV, YT_DLP_ENV, STATE_DIR_ENV] { + unsafe { std::env::remove_var(key) }; + } + guard + } + + /// Writes an executable stub that reports `version` when asked for `--version`. + fn fake_yt_dlp(path: &Path, version: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, format!("#!/bin/sh\necho {version}\n")).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + + #[test] + fn resolve_yt_dlp_prefers_state_dir_when_newer() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let pinned = tmp.path().join("nix/yt-dlp"); + fake_yt_dlp(&pinned, "2026.08.19"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.09.01"); + + unsafe { + std::env::set_var(YT_DLP_ENV, &pinned); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), state.join("yt-dlp/yt-dlp")); + } + + #[test] + fn resolve_yt_dlp_prefers_pinned_when_newer() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let pinned = tmp.path().join("nix/yt-dlp"); + fake_yt_dlp(&pinned, "2026.09.15"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.08.19"); + + unsafe { + std::env::set_var(YT_DLP_ENV, &pinned); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), pinned); + } + + #[test] + fn resolve_yt_dlp_breaks_version_ties_toward_state_dir() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let pinned = tmp.path().join("nix/yt-dlp"); + fake_yt_dlp(&pinned, "2026.09.01"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.09.01"); + + unsafe { + std::env::set_var(YT_DLP_ENV, &pinned); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), state.join("yt-dlp/yt-dlp")); + } + + #[test] + fn resolve_yt_dlp_honours_force_override_regardless_of_version() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let forced = tmp.path().join("forced/yt-dlp"); + fake_yt_dlp(&forced, "2020.01.01"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.09.01"); + + unsafe { + std::env::set_var(YT_DLP_FORCE_ENV, &forced); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), forced); + } + + #[test] + fn resolve_yt_dlp_falls_back_to_bare_when_no_candidate_exists() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + unsafe { + std::env::set_var(YT_DLP_ENV, tmp.path().join("missing/yt-dlp")); + std::env::set_var(STATE_DIR_ENV, tmp.path().join("empty-state")); + } + + assert_eq!(resolve_yt_dlp_uncached(), PathBuf::from("yt-dlp")); + } + + #[test] + fn state_dir_override_wins_over_platform_default() { + let _guard = env_guard(); + unsafe { std::env::set_var(STATE_DIR_ENV, "/tmp/archivr-state-override") }; + assert_eq!(state_dir(), Some(PathBuf::from("/tmp/archivr-state-override"))); + } #[test] fn quality_format_audio() { From 585d0a049b1ba7951b1ac474655b8b683778c588 Mon Sep 17 00:00:00 2001 From: archivr-qa Date: Sun, 23 Aug 2026 19:22:45 +0200 Subject: [PATCH 2/2] feat(cli): add `archivr yt-dlp update|status` subcommand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `update` fetches the latest release tag from the GitHub API (or takes --version), downloads the cross-platform python zipapp, and installs it into archivr's state dir. The install is atomic — staged as yt-dlp.new, chmod +x'd, then renamed over the target — so a concurrently running capture never sees a half-written binary. A sibling .version file makes a repeat update a no-op instead of a 3MB re-download. The download is checked for the python3 shebang before install, which catches the usual failure mode of getting an HTML error page back. python3 itself is only warned about, not required: the server may run under a nix wrapper with its own PATH. `status` prints all three candidates (env / state-dir / PATH fallback) with their versions and stars whichever the resolver picks, so it is obvious which yt-dlp a capture will actually use. reqwest is pulled from the existing workspace dependency; the GitHub JSON is parsed with serde_json so the "json" feature is not needed. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 1 + crates/archivr-cli/Cargo.toml | 1 + crates/archivr-cli/src/main.rs | 197 ++++++++++++++++++++++++++++++++- 3 files changed, 195 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c512560..ac0434c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,7 @@ dependencies = [ "chrono", "clap", "regex", + "reqwest", "rusqlite", "serde_json", ] diff --git a/crates/archivr-cli/Cargo.toml b/crates/archivr-cli/Cargo.toml index 3a190d7..6f0d0bf 100644 --- a/crates/archivr-cli/Cargo.toml +++ b/crates/archivr-cli/Cargo.toml @@ -15,3 +15,4 @@ clap.workspace = true regex.workspace = true rusqlite.workspace = true serde_json.workspace = true +reqwest.workspace = true diff --git a/crates/archivr-cli/src/main.rs b/crates/archivr-cli/src/main.rs index ec10543..83dd947 100644 --- a/crates/archivr-cli/src/main.rs +++ b/crates/archivr-cli/src/main.rs @@ -1,12 +1,27 @@ -use anyhow::{Context, Result}; -use archivr_core::{archive, capture::CaptureConfig}; +use anyhow::{bail, Context, Result}; +use archivr_core::{ + archive, + capture::CaptureConfig, + downloader::ytdlp::{ + pinned_yt_dlp, probe_version, resolve_yt_dlp, state_dir, state_dir_yt_dlp, + }, +}; use clap::{Parser, Subcommand}; use std::{ env, - path::Path, + path::{Path, PathBuf}, process, + process::Command as ProcCommand, }; +/// GitHub release metadata endpoint for the upstream yt-dlp project. +const YT_DLP_LATEST_RELEASE: &str = + "https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest"; + +/// Every python zipapp starts with this shebang; used as a sanity check that we +/// downloaded the artifact and not an HTML error page or an LFS pointer. +const ZIPAPP_SHEBANG: &[u8] = b"#!/usr/bin/env python3"; + #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { @@ -48,6 +63,25 @@ enum Command { #[arg(long = "force-with-info-removal")] force_with_info_removal: bool, }, + + /// Inspect or update the yt-dlp binary archivr runs + #[command(name = "yt-dlp")] + YtDlp { + #[command(subcommand)] + subcmd: YtDlpCmd, + }, +} + +#[derive(Subcommand, Debug)] +enum YtDlpCmd { + /// Download the latest yt-dlp zipapp into archivr's state directory + Update { + /// Install this exact release tag instead of the latest (e.g. 2026.09.15) + #[arg(long)] + version: Option, + }, + /// Show every yt-dlp candidate, its version, and which one wins + Status, } fn main() -> Result<()> { @@ -96,7 +130,162 @@ fn main() -> Result<()> { ); Ok(()) - } // _ => eprintln!("Unknown command: {:?}", args.command), + } + + Command::YtDlp { subcmd } => match subcmd { + YtDlpCmd::Update { version } => yt_dlp_update(version.as_deref()), + YtDlpCmd::Status => yt_dlp_status(), + }, } } + +/// Resolves `/yt-dlp/`, erroring out if there is no usable HOME. +fn yt_dlp_state_dir() -> Result { + state_dir() + .map(|d| d.join("yt-dlp")) + .context("could not determine a state directory (is $HOME set?)") +} + +/// Renders one `status` row. Missing candidates show an em dash. +fn status_row(role: &str, path: Option<&Path>, chosen: &Path) { + match path { + Some(p) => { + let version = probe_version(p).unwrap_or_else(|| "—".to_string()); + let star = if p == chosen { "*" } else { "" }; + println!("{role}\t{}\t{version}\t{star}", p.display()); + } + None => println!("{role}\t—\t—\t"), + } +} + +fn yt_dlp_status() -> Result<()> { + let chosen = resolve_yt_dlp(); + + println!("role\tpath\tversion\tchosen"); + status_row("env (ARCHIVR_YT_DLP)", pinned_yt_dlp().as_deref(), &chosen); + + // Show the state-dir slot even when empty, so users can see where an + // `archivr yt-dlp update` would land. + let state_candidate = state_dir_yt_dlp().filter(|p| p.is_file()); + status_row("state-dir", state_candidate.as_deref(), &chosen); + + status_row( + "path-fallback (yt-dlp)", + Some(Path::new("yt-dlp")), + &chosen, + ); + + if let Ok(dir) = yt_dlp_state_dir() { + if state_dir_yt_dlp().is_none_or(|p| !p.is_file()) { + println!("\nNo state-dir install yet; `archivr yt-dlp update` would write to {}", dir.join("yt-dlp").display()); + } + } + + Ok(()) +} + +/// Asks the GitHub API for the newest yt-dlp release tag. +fn latest_yt_dlp_version(client: &reqwest::blocking::Client) -> Result { + let body = client + .get(YT_DLP_LATEST_RELEASE) + .send() + .context("failed to reach the GitHub releases API")? + .error_for_status() + .context("GitHub releases API returned an error")? + .text() + .context("failed to read the GitHub releases API response")?; + + let json: serde_json::Value = + serde_json::from_str(&body).context("GitHub releases API returned invalid JSON")?; + + json.get("tag_name") + .and_then(|t| t.as_str()) + .map(str::to_string) + .context("GitHub releases API response had no tag_name") +} + +fn yt_dlp_update(requested_version: Option<&str>) -> Result<()> { + let dir = yt_dlp_state_dir()?; + let target = dir.join("yt-dlp"); + let staging = dir.join("yt-dlp.new"); + let version_file = dir.join(".version"); + + let client = reqwest::blocking::Client::builder() + .user_agent(concat!("archivr-cli/", env!("CARGO_PKG_VERSION"))) + .build() + .context("failed to build an HTTP client")?; + + let version = match requested_version { + Some(v) => v.to_string(), + None => latest_yt_dlp_version(&client)?, + }; + + // The sibling .version file is what lets us skip a ~3MB download on a + // no-op update; the binary itself is a zipapp with no cheap version probe + // that doesn't cost a python startup. + let installed = std::fs::read_to_string(&version_file).ok(); + if target.is_file() && installed.as_deref().map(str::trim) == Some(version.as_str()) { + println!("yt-dlp {version} is already installed at {}", target.display()); + return Ok(()); + } + + println!("Downloading yt-dlp {version}…"); + let url = format!("https://github.com/yt-dlp/yt-dlp/releases/download/{version}/yt-dlp"); + let bytes = client + .get(&url) + .send() + .with_context(|| format!("failed to download {url}"))? + .error_for_status() + .with_context(|| format!("download failed — is {version} a real release tag?"))? + .bytes() + .context("failed to read the downloaded yt-dlp body")?; + + if !bytes.starts_with(ZIPAPP_SHEBANG) { + bail!( + "downloaded artifact from {url} is not a python zipapp \ + (expected it to start with `{}`) — refusing to install it", + String::from_utf8_lossy(ZIPAPP_SHEBANG) + ); + } + + std::fs::create_dir_all(&dir) + .with_context(|| format!("failed to create {}", dir.display()))?; + std::fs::write(&staging, &bytes) + .with_context(|| format!("failed to write {}", staging.display()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755)) + .with_context(|| format!("failed to chmod +x {}", staging.display()))?; + } + + // Atomic swap: a concurrently-running archivr sees either the whole old + // binary or the whole new one, never a half-written file. + std::fs::rename(&staging, &target) + .with_context(|| format!("failed to install {}", target.display()))?; + std::fs::write(&version_file, format!("{version}\n")) + .with_context(|| format!("failed to record version in {}", version_file.display()))?; + + // The zipapp is python source, not a native binary — installing it on a + // host without python3 is legal (the server may run under a nix wrapper + // with its own PATH) but worth flagging loudly. + let has_python = ProcCommand::new("python3") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if !has_python { + eprintln!( + "warning: python3 was not found on PATH — the yt-dlp zipapp just installed \ + at {} will not run until python3 is available", + target.display() + ); + } + + println!("Installed yt-dlp {version} to {}", target.display()); + println!("archivr will now prefer it whenever it is newer than the pinned binary (ARCHIVR_YT_DLP)."); + + Ok(()) +}