diff --git a/Cargo.lock b/Cargo.lock index c512560..ac0434c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,7 @@ dependencies = [ "chrono", "clap", "regex", + "reqwest", "rusqlite", "serde_json", ] diff --git a/crates/archivr-cli/Cargo.toml b/crates/archivr-cli/Cargo.toml index 3a190d7..6f0d0bf 100644 --- a/crates/archivr-cli/Cargo.toml +++ b/crates/archivr-cli/Cargo.toml @@ -15,3 +15,4 @@ clap.workspace = true regex.workspace = true rusqlite.workspace = true serde_json.workspace = true +reqwest.workspace = true diff --git a/crates/archivr-cli/src/main.rs b/crates/archivr-cli/src/main.rs index ec10543..83dd947 100644 --- a/crates/archivr-cli/src/main.rs +++ b/crates/archivr-cli/src/main.rs @@ -1,12 +1,27 @@ -use anyhow::{Context, Result}; -use archivr_core::{archive, capture::CaptureConfig}; +use anyhow::{bail, Context, Result}; +use archivr_core::{ + archive, + capture::CaptureConfig, + downloader::ytdlp::{ + pinned_yt_dlp, probe_version, resolve_yt_dlp, state_dir, state_dir_yt_dlp, + }, +}; use clap::{Parser, Subcommand}; use std::{ env, - path::Path, + path::{Path, PathBuf}, process, + process::Command as ProcCommand, }; +/// GitHub release metadata endpoint for the upstream yt-dlp project. +const YT_DLP_LATEST_RELEASE: &str = + "https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest"; + +/// Every python zipapp starts with this shebang; used as a sanity check that we +/// downloaded the artifact and not an HTML error page or an LFS pointer. +const ZIPAPP_SHEBANG: &[u8] = b"#!/usr/bin/env python3"; + #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { @@ -48,6 +63,25 @@ enum Command { #[arg(long = "force-with-info-removal")] force_with_info_removal: bool, }, + + /// Inspect or update the yt-dlp binary archivr runs + #[command(name = "yt-dlp")] + YtDlp { + #[command(subcommand)] + subcmd: YtDlpCmd, + }, +} + +#[derive(Subcommand, Debug)] +enum YtDlpCmd { + /// Download the latest yt-dlp zipapp into archivr's state directory + Update { + /// Install this exact release tag instead of the latest (e.g. 2026.09.15) + #[arg(long)] + version: Option, + }, + /// Show every yt-dlp candidate, its version, and which one wins + Status, } fn main() -> Result<()> { @@ -96,7 +130,162 @@ fn main() -> Result<()> { ); Ok(()) - } // _ => eprintln!("Unknown command: {:?}", args.command), + } + + Command::YtDlp { subcmd } => match subcmd { + YtDlpCmd::Update { version } => yt_dlp_update(version.as_deref()), + YtDlpCmd::Status => yt_dlp_status(), + }, } } + +/// Resolves `/yt-dlp/`, erroring out if there is no usable HOME. +fn yt_dlp_state_dir() -> Result { + state_dir() + .map(|d| d.join("yt-dlp")) + .context("could not determine a state directory (is $HOME set?)") +} + +/// Renders one `status` row. Missing candidates show an em dash. +fn status_row(role: &str, path: Option<&Path>, chosen: &Path) { + match path { + Some(p) => { + let version = probe_version(p).unwrap_or_else(|| "—".to_string()); + let star = if p == chosen { "*" } else { "" }; + println!("{role}\t{}\t{version}\t{star}", p.display()); + } + None => println!("{role}\t—\t—\t"), + } +} + +fn yt_dlp_status() -> Result<()> { + let chosen = resolve_yt_dlp(); + + println!("role\tpath\tversion\tchosen"); + status_row("env (ARCHIVR_YT_DLP)", pinned_yt_dlp().as_deref(), &chosen); + + // Show the state-dir slot even when empty, so users can see where an + // `archivr yt-dlp update` would land. + let state_candidate = state_dir_yt_dlp().filter(|p| p.is_file()); + status_row("state-dir", state_candidate.as_deref(), &chosen); + + status_row( + "path-fallback (yt-dlp)", + Some(Path::new("yt-dlp")), + &chosen, + ); + + if let Ok(dir) = yt_dlp_state_dir() { + if state_dir_yt_dlp().is_none_or(|p| !p.is_file()) { + println!("\nNo state-dir install yet; `archivr yt-dlp update` would write to {}", dir.join("yt-dlp").display()); + } + } + + Ok(()) +} + +/// Asks the GitHub API for the newest yt-dlp release tag. +fn latest_yt_dlp_version(client: &reqwest::blocking::Client) -> Result { + let body = client + .get(YT_DLP_LATEST_RELEASE) + .send() + .context("failed to reach the GitHub releases API")? + .error_for_status() + .context("GitHub releases API returned an error")? + .text() + .context("failed to read the GitHub releases API response")?; + + let json: serde_json::Value = + serde_json::from_str(&body).context("GitHub releases API returned invalid JSON")?; + + json.get("tag_name") + .and_then(|t| t.as_str()) + .map(str::to_string) + .context("GitHub releases API response had no tag_name") +} + +fn yt_dlp_update(requested_version: Option<&str>) -> Result<()> { + let dir = yt_dlp_state_dir()?; + let target = dir.join("yt-dlp"); + let staging = dir.join("yt-dlp.new"); + let version_file = dir.join(".version"); + + let client = reqwest::blocking::Client::builder() + .user_agent(concat!("archivr-cli/", env!("CARGO_PKG_VERSION"))) + .build() + .context("failed to build an HTTP client")?; + + let version = match requested_version { + Some(v) => v.to_string(), + None => latest_yt_dlp_version(&client)?, + }; + + // The sibling .version file is what lets us skip a ~3MB download on a + // no-op update; the binary itself is a zipapp with no cheap version probe + // that doesn't cost a python startup. + let installed = std::fs::read_to_string(&version_file).ok(); + if target.is_file() && installed.as_deref().map(str::trim) == Some(version.as_str()) { + println!("yt-dlp {version} is already installed at {}", target.display()); + return Ok(()); + } + + println!("Downloading yt-dlp {version}…"); + let url = format!("https://github.com/yt-dlp/yt-dlp/releases/download/{version}/yt-dlp"); + let bytes = client + .get(&url) + .send() + .with_context(|| format!("failed to download {url}"))? + .error_for_status() + .with_context(|| format!("download failed — is {version} a real release tag?"))? + .bytes() + .context("failed to read the downloaded yt-dlp body")?; + + if !bytes.starts_with(ZIPAPP_SHEBANG) { + bail!( + "downloaded artifact from {url} is not a python zipapp \ + (expected it to start with `{}`) — refusing to install it", + String::from_utf8_lossy(ZIPAPP_SHEBANG) + ); + } + + std::fs::create_dir_all(&dir) + .with_context(|| format!("failed to create {}", dir.display()))?; + std::fs::write(&staging, &bytes) + .with_context(|| format!("failed to write {}", staging.display()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755)) + .with_context(|| format!("failed to chmod +x {}", staging.display()))?; + } + + // Atomic swap: a concurrently-running archivr sees either the whole old + // binary or the whole new one, never a half-written file. + std::fs::rename(&staging, &target) + .with_context(|| format!("failed to install {}", target.display()))?; + std::fs::write(&version_file, format!("{version}\n")) + .with_context(|| format!("failed to record version in {}", version_file.display()))?; + + // The zipapp is python source, not a native binary — installing it on a + // host without python3 is legal (the server may run under a nix wrapper + // with its own PATH) but worth flagging loudly. + let has_python = ProcCommand::new("python3") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if !has_python { + eprintln!( + "warning: python3 was not found on PATH — the yt-dlp zipapp just installed \ + at {} will not run until python3 is available", + target.display() + ); + } + + println!("Installed yt-dlp {version} to {}", target.display()); + println!("archivr will now prefer it whenever it is newer than the pinned binary (ARCHIVR_YT_DLP)."); + + Ok(()) +} diff --git a/crates/archivr-core/src/downloader/ytdlp.rs b/crates/archivr-core/src/downloader/ytdlp.rs index 26c091d..e55943d 100644 --- a/crates/archivr-core/src/downloader/ytdlp.rs +++ b/crates/archivr-core/src/downloader/ytdlp.rs @@ -4,6 +4,7 @@ use std::{ env, path::{Path, PathBuf}, process::Command, + sync::OnceLock, }; use uuid::Uuid; use serde_json; @@ -11,6 +12,109 @@ use serde_json; use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file}; use crate::hash::hash_file; +/// Env var that force-pins a specific yt-dlp binary, bypassing version comparison. +pub const YT_DLP_FORCE_ENV: &str = "ARCHIVR_YT_DLP_FORCE"; +/// Env var set by the nix flake wrapper, pointing at the pinned yt-dlp. +pub const YT_DLP_ENV: &str = "ARCHIVR_YT_DLP"; +/// Override for the mutable state directory (used by `archivr yt-dlp` and tests). +pub const STATE_DIR_ENV: &str = "ARCHIVR_STATE_DIR"; + +static RESOLVED_YT_DLP: OnceLock = OnceLock::new(); + +/// Mutable per-user state directory for archivr. +/// +/// `ARCHIVR_STATE_DIR` wins if set. Otherwise this mirrors what `dirs::state_dir()` +/// would give us without taking on the dependency: `~/Library/Application Support` +/// on macOS, `$XDG_STATE_HOME` (default `~/.local/state`) elsewhere. +pub fn state_dir() -> Option { + if let Some(dir) = env::var_os(STATE_DIR_ENV) { + if !dir.is_empty() { + return Some(PathBuf::from(dir)); + } + } + + let home = PathBuf::from(env::var_os("HOME").filter(|h| !h.is_empty())?); + + if cfg!(target_os = "macos") { + Some(home.join("Library").join("Application Support").join("archivr")) + } else { + let base = env::var_os("XDG_STATE_HOME") + .filter(|d| !d.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".local").join("state")); + Some(base.join("archivr")) + } +} + +/// Path of the user-installed (self-updated) yt-dlp inside the state dir. +pub fn state_dir_yt_dlp() -> Option { + state_dir().map(|d| d.join("yt-dlp").join("yt-dlp")) +} + +/// The nix-pinned yt-dlp advertised via `ARCHIVR_YT_DLP`, if it exists on disk. +pub fn pinned_yt_dlp() -> Option { + let p = PathBuf::from(env::var_os(YT_DLP_ENV).filter(|v| !v.is_empty())?); + p.is_file().then_some(p) +} + +/// Runs ` --version` and returns the trimmed stdout. +/// +/// yt-dlp versions are `YYYY.MM.DD`, so plain string ordering is chronological +/// ordering — no semver parsing needed. +pub fn probe_version(binary: &Path) -> Option { + let out = Command::new(binary).arg("--version").output().ok()?; + if !out.status.success() { + return None; + } + let version = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (!version.is_empty()).then_some(version) +} + +/// The candidate yt-dlp binaries, in priority order for tie-breaking +/// (later entries win ties, so the deliberately-installed state-dir copy is last). +pub fn yt_dlp_candidates() -> Vec<(&'static str, PathBuf)> { + let mut candidates = Vec::new(); + if let Some(p) = pinned_yt_dlp() { + candidates.push(("env (ARCHIVR_YT_DLP)", p)); + } + if let Some(p) = state_dir_yt_dlp() { + if p.is_file() { + candidates.push(("state-dir", p)); + } + } + candidates +} + +/// Picks the yt-dlp binary to run, without consulting the process-wide cache. +/// +/// Priority: `ARCHIVR_YT_DLP_FORCE` > newest of (pinned, state-dir) by version +/// string > bare `yt-dlp` (PATH lookup, the historical behaviour). +pub fn resolve_yt_dlp_uncached() -> PathBuf { + if let Some(forced) = env::var_os(YT_DLP_FORCE_ENV).filter(|v| !v.is_empty()) { + let forced = PathBuf::from(forced); + if forced.is_file() { + return forced; + } + } + + yt_dlp_candidates() + .into_iter() + .filter_map(|(_, path)| probe_version(&path).map(|v| (v, path))) + // `max_by` keeps the *last* maximum, and the state-dir candidate is last, + // so an exact version tie resolves in favour of the user's own install. + .max_by(|(a, _), (b, _)| a.cmp(b)) + .map(|(_, path)| path) + .unwrap_or_else(|| PathBuf::from("yt-dlp")) +} + +/// Cached [`resolve_yt_dlp_uncached`] — `--version` is spawned at most once +/// per process no matter how many yt-dlp calls the run makes. +pub fn resolve_yt_dlp() -> PathBuf { + RESOLVED_YT_DLP + .get_or_init(resolve_yt_dlp_uncached) + .clone() +} + /// A single item in a flat playlist listing from `fetch_playlist_info`. #[derive(Debug)] pub struct PlaylistItem { @@ -164,7 +268,7 @@ pub fn download( ) -> Result<(String, String)> { println!("Downloading with yt-dlp: {path}"); - let ytdlp = env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); let is_audio = quality == Some("audio"); let temp_dir = store_path.join("temp").join(timestamp); @@ -207,7 +311,7 @@ pub fn download( .arg("-o") .arg(&out_template) .output() - .with_context(|| format!("failed to spawn {ytdlp} process")); + .with_context(|| format!("failed to spawn {} process", ytdlp.display())); // Remove cookie file immediately regardless of outcome. if let Some(cf) = &cookie_file { @@ -253,7 +357,7 @@ fn find_downloaded_file(temp_dir: &Path, timestamp: &str) -> Result { /// On failure (non-zero exit or no stdout), prints the captured stderr /// to stderr (for debugging) then returns `None` so callers can proceed. pub fn fetch_metadata(path: &str, cookies: &HashMap) -> Option { - let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); // Write a temp cookie file if needed; UUID-named to avoid collisions. let cookie_file: Option = if !cookies.is_empty() { @@ -342,7 +446,7 @@ fn normalize_item_url( /// Returns an error if yt-dlp fails, the output is not valid JSON, or /// the root `_type` is not `"playlist"`. pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Result { - let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); let cookie_file: Option = if !cookies.is_empty() { let domain = domain_from_url(url); @@ -366,7 +470,7 @@ pub fn fetch_playlist_info(url: &str, cookies: &HashMap) -> Resu if let Some(cf) = &cookie_file { let _ = std::fs::remove_file(cf); } - let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?; + let out = out.with_context(|| format!("failed to spawn {}", ytdlp.display()))?; if !out.status.success() { let stderr = String::from_utf8_lossy(&out.stderr); bail!("yt-dlp -J --flat-playlist failed for {url}: {stderr}"); @@ -425,7 +529,7 @@ pub fn probe_playlist_qualities( url: &str, cookies: &HashMap, ) -> Result { - let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); + let ytdlp = resolve_yt_dlp(); let cookie_file: Option = if !cookies.is_empty() { let domain = domain_from_url(url); @@ -449,7 +553,7 @@ pub fn probe_playlist_qualities( if let Some(cf) = &cookie_file { let _ = std::fs::remove_file(cf); } - let out = out.with_context(|| format!("failed to spawn {ytdlp}"))?; + let out = out.with_context(|| format!("failed to spawn {}", ytdlp.display()))?; if !out.status.success() { let stderr = String::from_utf8_lossy(&out.stderr); bail!("yt-dlp -J failed for {url}: {stderr}"); @@ -501,7 +605,131 @@ pub fn probe_playlist_qualities( #[cfg(test)] mod tests { - use super::{available_video_heights, has_audio_track, quality_format}; + use super::{ + available_video_heights, has_audio_track, quality_format, resolve_yt_dlp_uncached, + state_dir, STATE_DIR_ENV, YT_DLP_ENV, YT_DLP_FORCE_ENV, + }; + use std::path::{Path, PathBuf}; + use std::sync::{Mutex, MutexGuard}; + + /// Env vars are process-global, so resolver tests take turns. + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + /// Clears every env var the resolver reads and hands back the serialising guard. + fn env_guard() -> MutexGuard<'static, ()> { + let guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for key in [YT_DLP_FORCE_ENV, YT_DLP_ENV, STATE_DIR_ENV] { + unsafe { std::env::remove_var(key) }; + } + guard + } + + /// Writes an executable stub that reports `version` when asked for `--version`. + fn fake_yt_dlp(path: &Path, version: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, format!("#!/bin/sh\necho {version}\n")).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + + #[test] + fn resolve_yt_dlp_prefers_state_dir_when_newer() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let pinned = tmp.path().join("nix/yt-dlp"); + fake_yt_dlp(&pinned, "2026.08.19"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.09.01"); + + unsafe { + std::env::set_var(YT_DLP_ENV, &pinned); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), state.join("yt-dlp/yt-dlp")); + } + + #[test] + fn resolve_yt_dlp_prefers_pinned_when_newer() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let pinned = tmp.path().join("nix/yt-dlp"); + fake_yt_dlp(&pinned, "2026.09.15"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.08.19"); + + unsafe { + std::env::set_var(YT_DLP_ENV, &pinned); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), pinned); + } + + #[test] + fn resolve_yt_dlp_breaks_version_ties_toward_state_dir() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let pinned = tmp.path().join("nix/yt-dlp"); + fake_yt_dlp(&pinned, "2026.09.01"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.09.01"); + + unsafe { + std::env::set_var(YT_DLP_ENV, &pinned); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), state.join("yt-dlp/yt-dlp")); + } + + #[test] + fn resolve_yt_dlp_honours_force_override_regardless_of_version() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + let forced = tmp.path().join("forced/yt-dlp"); + fake_yt_dlp(&forced, "2020.01.01"); + + let state = tmp.path().join("state"); + fake_yt_dlp(&state.join("yt-dlp/yt-dlp"), "2026.09.01"); + + unsafe { + std::env::set_var(YT_DLP_FORCE_ENV, &forced); + std::env::set_var(STATE_DIR_ENV, &state); + } + + assert_eq!(resolve_yt_dlp_uncached(), forced); + } + + #[test] + fn resolve_yt_dlp_falls_back_to_bare_when_no_candidate_exists() { + let _guard = env_guard(); + let tmp = tempfile::tempdir().unwrap(); + + unsafe { + std::env::set_var(YT_DLP_ENV, tmp.path().join("missing/yt-dlp")); + std::env::set_var(STATE_DIR_ENV, tmp.path().join("empty-state")); + } + + assert_eq!(resolve_yt_dlp_uncached(), PathBuf::from("yt-dlp")); + } + + #[test] + fn state_dir_override_wins_over_platform_default() { + let _guard = env_guard(); + unsafe { std::env::set_var(STATE_DIR_ENV, "/tmp/archivr-state-override") }; + assert_eq!(state_dir(), Some(PathBuf::from("/tmp/archivr-state-override"))); + } #[test] fn quality_format_audio() {