1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

feat: add video quality selection for yt-dlp captures (#17)

- ytdlp::download() accepts quality: Option<&str>; quality_format()
  maps best/1080p/720p/480p/360p to yt-dlp -f format strings
- perform_capture() threads quality through to the downloader
- CaptureBody gains optional quality field; capture_handler validates
  it against the allowlist (400 on unknown values) before spawning
- CLI passes None (preserves existing best-quality behaviour)
- Frontend: isVideoSource() mirrors determine_source() exactly —
  shows quality picker only for yt-dlp-backed sources, excludes
  playlist/channel shorthands and tweet/thread paths
- submitCapture(archiveId, locator, quality) sends quality in POST body
- CSS: .capture-quality styles the inline select to fit the capture row
- Tests: quality_format unit tests in ytdlp.rs; two new route tests
  (valid quality accepted, invalid quality rejected with 400)
- Docs: video quality section added under Supported Platforms
This commit is contained in:
TheGeneralist 2026-07-05 20:42:41 +02:00 committed by GitHub
parent 84c5730b6a
commit b8e496457f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 689 additions and 89 deletions

View file

@ -63,7 +63,7 @@ fn main() -> Result<()> {
} }
}; };
let archive_paths = archive::read_archive_paths(&archive_path)?; let archive_paths = archive::read_archive_paths(&archive_path)?;
let result = archivr_core::capture::perform_capture(&archive_paths, path, None)?; let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None)?;
println!("Archived: run {}", result.run_uid); println!("Archived: run {}", result.run_uid);
Ok(()) Ok(())
} }

View file

@ -331,6 +331,26 @@ fn determine_source(path: &str) -> Source {
Source::Other Source::Other
} }
/// Resolves `locator` to the canonical URL that yt-dlp should receive, or
/// returns `None` if the locator does not map to a yt-dlp-downloadable source
/// (e.g. tweet/thread shorthands, web pages, local files, playlists/channels).
///
/// Use this to gate the probe endpoint: only call `fetch_metadata` when this
/// returns `Some`.
pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
let source = determine_source(locator);
match source {
Source::YouTubeVideo
| Source::X
| Source::Instagram
| Source::Facebook
| Source::TikTok
| Source::Reddit
| Source::Snapchat => Some(expand_shorthand_to_url(locator, &source)),
_ => None,
}
}
fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result<bool> { fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result<bool> {
let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?);
Ok(path.exists()) Ok(path.exists())
@ -427,20 +447,6 @@ fn local_file_extension(path: &str) -> String {
.map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy()))
} }
fn media_file_extension(source: Source, path: &str) -> String {
match source {
Source::YouTubeVideo
| Source::X
| Source::Instagram
| Source::Facebook
| Source::TikTok
| Source::Reddit
| Source::Snapchat => ".mp4".to_string(),
Source::Local => local_file_extension(path),
_ => String::new(),
}
}
fn tweet_id_from_archive_path(path: &str) -> Option<String> { fn tweet_id_from_archive_path(path: &str) -> Option<String> {
path.split(':').next_back().and_then(parse_tweet_id) path.split(':').next_back().and_then(parse_tweet_id)
} }
@ -677,6 +683,7 @@ pub fn perform_capture(
archive_paths: &ArchivePaths, archive_paths: &ArchivePaths,
locator: &str, locator: &str,
archive_id: Option<&str>, archive_id: Option<&str>,
quality: Option<&str>,
) -> Result<CaptureResult> { ) -> Result<CaptureResult> {
// Append a UUID so parallel captures starting in the same millisecond // Append a UUID so parallel captures starting in the same millisecond
// never collide on the staging directory or file names. // never collide on the staging directory or file names.
@ -1015,7 +1022,7 @@ pub fn perform_capture(
_ => None, _ => None,
}; };
let hash = match source { let (hash, file_extension) = match source {
Source::YouTubeVideo Source::YouTubeVideo
| Source::X | Source::X
| Source::Instagram | Source::Instagram
@ -1023,8 +1030,8 @@ pub fn perform_capture(
| Source::TikTok | Source::TikTok
| Source::Reddit | Source::Reddit
| Source::Snapchat => { | Source::Snapchat => {
match downloader::ytdlp::download(path.clone(), store_path, &timestamp) { match downloader::ytdlp::download(path.clone(), store_path, &timestamp, quality) {
Ok(h) => h, Ok(result) => result,
Err(e) => { Err(e) => {
return Err(fail_run( return Err(fail_run(
&conn, &conn,
@ -1037,7 +1044,7 @@ pub fn perform_capture(
} }
Source::Local => { Source::Local => {
match downloader::local::save(path.clone(), store_path, &timestamp) { match downloader::local::save(path.clone(), store_path, &timestamp) {
Ok(h) => h, Ok(h) => (h, local_file_extension(&path)),
Err(e) => { Err(e) => {
return Err(fail_run( return Err(fail_run(
&conn, &conn,
@ -1058,8 +1065,6 @@ pub fn perform_capture(
} }
_ => unreachable!(), _ => unreachable!(),
}; };
let file_extension = media_file_extension(source, &path);
let temp_file = store_path let temp_file = store_path
.join("temp") .join("temp")
.join(&timestamp) .join(&timestamp)

View file

@ -1,26 +1,126 @@
use anyhow::{Context, Result, bail}; use anyhow::{bail, Context, Result};
use std::{env, path::Path, process::Command}; use std::{env, path::{Path, PathBuf}, process::Command};
use crate::hash::hash_file; use crate::hash::hash_file;
pub fn download(path: String, store_path: &Path, timestamp: &String) -> Result<String> { /// Returns the yt-dlp `-f` format selector for `quality`.
///
/// - `"audio"` → prefers native Opus/WebM (most efficient), then native
/// AAC/M4A, then any best-audio fallback — no transcoding, smallest file
/// at equivalent perceptual quality.
/// - `"NNNp"` (e.g. `"1080p"`) → height-capped selector with `/best` fallback
/// - `None` / `"best"` / anything else → highest-quality video+audio
pub fn quality_format(quality: Option<&str>) -> String {
if quality == Some("audio") {
// Opus (WebM) is more efficient than AAC (M4A) at the same perceptual
// quality, so prefer it first. Both are taken natively — no transcode.
return "bestaudio[ext=webm]/bestaudio[ext=m4a]/bestaudio/best".to_string();
}
if let Some(q) = quality {
if let Some(h) = q.strip_suffix('p').and_then(|n| n.parse::<u32>().ok()) {
return format!("bestvideo[height<={h}]+bestaudio/best[height<={h}]/best");
}
}
"bestvideo+bestaudio/best".to_string()
}
/// Combined result of a yt-dlp metadata probe.
pub struct ProbeResult {
/// Distinct video heights available, sorted highest-first (e.g. `[1080, 720, 480]`).
pub video_heights: Vec<u32>,
/// True when at least one format with a real audio codec exists.
pub has_audio: bool,
}
/// Parses a yt-dlp `--dump-json` response into a `ProbeResult`.
pub fn probe_result(json: &str) -> ProbeResult {
ProbeResult {
video_heights: available_video_heights(json),
has_audio: has_audio_track(json),
}
}
/// Distinct video heights from a yt-dlp `--dump-json` response, sorted highest-first.
/// Audio-only formats (`vcodec == "none"`) and zero-height entries are excluded.
pub fn available_video_heights(json: &str) -> Vec<u32> {
let v: serde_json::Value = match serde_json::from_str(json) {
Ok(v) => v,
Err(_) => return vec![],
};
let Some(formats) = v.get("formats").and_then(|f| f.as_array()) else {
return vec![];
};
let mut heights: Vec<u32> = formats
.iter()
.filter_map(|f| {
let vcodec = f.get("vcodec")?.as_str()?;
if vcodec == "none" {
return None;
}
let h = f.get("height")?.as_u64()?;
if h == 0 { None } else { Some(h as u32) }
})
.collect();
heights.sort_unstable_by(|a, b| b.cmp(a));
heights.dedup();
heights
}
/// Returns true when the yt-dlp `--dump-json` response contains at least one
/// format with a real audio codec (i.e. `acodec != "none"`).
pub fn has_audio_track(json: &str) -> bool {
let Ok(v) = serde_json::from_str::<serde_json::Value>(json) else {
return false;
};
let Some(formats) = v.get("formats").and_then(|f| f.as_array()) else {
return false;
};
formats.iter().any(|f| {
f.get("acodec")
.and_then(|a| a.as_str())
.is_some_and(|a| a != "none")
})
}
/// Downloads `path` via yt-dlp and returns `(hash, file_extension_with_dot)`.
///
/// For video the extension is always `.mp4` (forced via `--merge-output-format`).
/// For audio (`quality == Some("audio")`) `-x` is passed to guarantee audio-only
/// output even when only combined A/V formats exist (yt-dlp strips the video
/// track). `--audio-format` is intentionally omitted so the audio stream is
/// remuxed into its native container without re-encoding — no lossy transcode,
/// no size inflation. The actual output extension is discovered by globbing.
pub fn download(
path: String,
store_path: &Path,
timestamp: &String,
quality: Option<&str>,
) -> Result<(String, String)> {
println!("Downloading with yt-dlp: {path}"); println!("Downloading with yt-dlp: {path}");
let ytdlp = env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); let ytdlp = env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
let is_audio = quality == Some("audio");
let temp_dir = store_path.join("temp").join(timestamp); let temp_dir = store_path.join("temp").join(timestamp);
std::fs::create_dir_all(&temp_dir)?; std::fs::create_dir_all(&temp_dir)?;
let out_file = temp_dir.join(format!("{timestamp}.mp4")); // %(ext)s lets yt-dlp write the correct extension for the chosen format.
let out_template = temp_dir.join(format!("{timestamp}.%(ext)s"));
let out = Command::new(&ytdlp) let mut cmd = Command::new(&ytdlp);
.arg(&path) cmd.arg(&path).arg("-f").arg(quality_format(quality));
.arg("-f") if is_audio {
.arg("bestvideo+bestaudio/best") // -x guarantees audio-only even when /best falls back to a combined
.arg("--merge-output-format") // A/V format. No --audio-format → native remux only, no re-encode.
.arg("mp4") cmd.arg("-x");
} else {
// Force the video container to mp4 so we always have a known extension.
cmd.arg("--merge-output-format").arg("mp4");
}
let out = cmd
.arg("-o") .arg("-o")
.arg(&out_file) .arg(&out_template)
.output() .output()
.with_context(|| format!("failed to spawn {ytdlp} process"))?; .with_context(|| format!("failed to spawn {ytdlp} process"))?;
@ -29,7 +129,31 @@ pub fn download(path: String, store_path: &Path, timestamp: &String) -> Result<S
bail!("yt-dlp failed: {stderr}"); bail!("yt-dlp failed: {stderr}");
} }
hash_file(&out_file) let actual_file = find_downloaded_file(&temp_dir, timestamp)?;
let ext = actual_file
.extension()
.map(|e| format!(".{}", e.to_string_lossy()))
.unwrap_or_default();
let hash = hash_file(&actual_file)?;
Ok((hash, ext))
}
/// Finds the file yt-dlp wrote to `temp_dir` whose stem is `timestamp`.
/// Ignores `.part` files (incomplete downloads).
fn find_downloaded_file(temp_dir: &Path, timestamp: &str) -> Result<PathBuf> {
let entries = std::fs::read_dir(temp_dir)
.with_context(|| format!("failed to read temp dir {}", temp_dir.display()))?;
for entry in entries.flatten() {
let name = entry.file_name();
let name_str = name.to_string_lossy();
if name_str.starts_with(timestamp) && !name_str.ends_with(".part") {
return Ok(entry.path());
}
}
bail!(
"yt-dlp output file not found in {}",
temp_dir.display()
)
} }
/// Fetches metadata JSON for `path` via `yt-dlp --dump-json`. /// Fetches metadata JSON for `path` via `yt-dlp --dump-json`.
@ -48,10 +172,90 @@ pub fn fetch_metadata(path: &str) -> Option<String> {
if !out.status.success() { if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr); let stderr = String::from_utf8_lossy(&out.stderr);
eprintln!("yt-dlp --dump-json failed for {path} (status {:?}): {stderr}", out.status); eprintln!(
"yt-dlp --dump-json failed for {path} (status {:?}): {stderr}",
out.status
);
return None; return None;
} }
let json = String::from_utf8(out.stdout).ok()?; let json = String::from_utf8(out.stdout).ok()?;
if json.trim().is_empty() { None } else { Some(json) } if json.trim().is_empty() { None } else { Some(json) }
} }
#[cfg(test)]
mod tests {
use super::{available_video_heights, has_audio_track, quality_format};
#[test]
fn quality_format_audio() {
assert_eq!(quality_format(Some("audio")), "bestaudio[ext=webm]/bestaudio[ext=m4a]/bestaudio/best");
}
#[test]
fn quality_format_known_heights() {
assert_eq!(
quality_format(Some("1080p")),
"bestvideo[height<=1080]+bestaudio/best[height<=1080]/best"
);
assert_eq!(
quality_format(Some("720p")),
"bestvideo[height<=720]+bestaudio/best[height<=720]/best"
);
assert_eq!(
quality_format(Some("2160p")),
"bestvideo[height<=2160]+bestaudio/best[height<=2160]/best"
);
}
#[test]
fn quality_format_defaults_to_best() {
assert_eq!(quality_format(None), "bestvideo+bestaudio/best");
assert_eq!(quality_format(Some("best")), "bestvideo+bestaudio/best");
assert_eq!(quality_format(Some("bogus")), "bestvideo+bestaudio/best");
}
#[test]
fn available_video_heights_parses_formats() {
let json = r#"{
"formats": [
{"height": 1080, "vcodec": "avc1.640028", "acodec": "none"},
{"height": 720, "vcodec": "avc1.4d401f", "acodec": "none"},
{"height": 1080, "vcodec": "avc1.640028", "acodec": "mp4a.40.2"},
{"height": null, "vcodec": "none", "acodec": "mp4a.40.2"},
{"height": 360, "vcodec": "none", "acodec": "mp4a.40.2"}
]
}"#;
assert_eq!(available_video_heights(json), vec![1080, 720]);
}
#[test]
fn available_video_heights_empty_on_audio_only() {
let json = r#"{"formats": [{"height": null, "vcodec": "none", "acodec": "mp4a.40.2"}]}"#;
assert_eq!(available_video_heights(json), vec![0u32; 0]);
}
#[test]
fn available_video_heights_empty_on_bad_json() {
assert_eq!(available_video_heights("not json"), vec![0u32; 0]);
assert_eq!(available_video_heights("{}"), vec![0u32; 0]);
}
#[test]
fn has_audio_track_detects_audio() {
let with_audio = r#"{"formats": [
{"vcodec": "avc1", "acodec": "mp4a.40.2"},
{"vcodec": "none", "acodec": "mp4a.40.2"}
]}"#;
assert!(has_audio_track(with_audio));
let video_only = r#"{"formats": [
{"vcodec": "avc1", "acodec": "none"}
]}"#;
assert!(!has_audio_track(video_only));
assert!(!has_audio_track("not json"));
assert!(!has_audio_track("{}"));
}
}

View file

@ -29,7 +29,7 @@ use std::{
}; };
use parking_lot::Mutex; use parking_lot::Mutex;
use archivr_core::{archive, capture, database}; use archivr_core::{archive, capture, database, downloader};
use axum::{ use axum::{
Json, Router, Json, Router,
extract::{ConnectInfo, Path, Query, Request, State}, extract::{ConnectInfo, Path, Query, Request, State},
@ -231,6 +231,7 @@ pub fn app_with_state(state: AppState) -> Router {
) )
.route("/api/archives/:archive_id/runs", get(list_runs)) .route("/api/archives/:archive_id/runs", get(list_runs))
.route("/api/archives/:archive_id/captures", post(capture_handler)) .route("/api/archives/:archive_id/captures", post(capture_handler))
.route("/api/archives/:archive_id/captures/probe", get(probe_handler))
.route( .route(
"/api/archives/:archive_id/capture_jobs/:job_uid", "/api/archives/:archive_id/capture_jobs/:job_uid",
get(get_capture_job_handler), get(get_capture_job_handler),
@ -637,6 +638,14 @@ async fn delete_entry_handler(
#[derive(Debug, serde::Deserialize)] #[derive(Debug, serde::Deserialize)]
struct CaptureBody { struct CaptureBody {
locator: String, locator: String,
/// Optional quality cap for yt-dlp sources: `"best"` or any `"NNNp"` string
/// (e.g. `"1080p"`, `"720p"`, `"2160p"`). Absent or `"best"` → highest available.
quality: Option<String>,
}
#[derive(Debug, serde::Deserialize)]
struct ProbeQuery {
locator: String,
} }
#[derive(Debug, serde::Deserialize)] #[derive(Debug, serde::Deserialize)]
@ -676,6 +685,18 @@ async fn capture_handler(
if body.locator.trim().is_empty() { if body.locator.trim().is_empty() {
return Err(ApiError::bad_request("locator must not be empty")); return Err(ApiError::bad_request("locator must not be empty"));
} }
if let Some(q) = &body.quality {
let valid = q == "best"
|| q == "audio"
|| q.strip_suffix('p')
.and_then(|n| n.parse::<u32>().ok())
.is_some();
if !valid {
return Err(ApiError::bad_request(
"invalid quality: must be \"best\", \"audio\", or a height string like \"1080p\"",
));
}
}
let mounted = mounted_archive(&state, &archive_id)?; let mounted = mounted_archive(&state, &archive_id)?;
let archive_paths = archive::read_archive_paths(&mounted.archive_path) let archive_paths = archive::read_archive_paths(&mounted.archive_path)
.map_err(ApiError::from)?; .map_err(ApiError::from)?;
@ -687,6 +708,7 @@ async fn capture_handler(
// Spawn background capture. // Spawn background capture.
let locator = body.locator.trim().to_string(); let locator = body.locator.trim().to_string();
let quality = body.quality.clone();
let archive_path = mounted.archive_path.clone(); let archive_path = mounted.archive_path.clone();
let job_uid_bg = job_uid.clone(); let job_uid_bg = job_uid.clone();
let archive_id_bg = archive_id.clone(); let archive_id_bg = archive_id.clone();
@ -699,7 +721,7 @@ async fn capture_handler(
} }
}; };
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok(); database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok();
match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg)) { match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref()) {
Ok(result) => { Ok(result) => {
database::update_capture_job_status( database::update_capture_job_status(
&conn, &conn,
@ -742,6 +764,64 @@ async fn get_capture_job_handler(
.ok_or_else(|| ApiError::not_found("capture job not found")) .ok_or_else(|| ApiError::not_found("capture job not found"))
} }
/// `GET /api/archives/:id/captures/probe?locator=<url>`
///
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
/// heights actually available at the given locator.
///
/// Response shapes:
/// - Locator is not a yt-dlp source (tweet, webpage, local, …):
/// `{ "has_video": false, "qualities": [] }` — 200
/// - yt-dlp ran and found no video tracks (e.g. tweet URL with no media):
/// `{ "has_video": false, "qualities": [] }` — 200
/// - yt-dlp ran and found video tracks:
/// `{ "has_video": true, "qualities": ["1080p", "720p", …] }` — 200
/// - yt-dlp itself failed (non-zero exit, network error, rate-limit, …):
/// 502 — caller should treat this as "probe inconclusive", not "no video"
async fn probe_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path(archive_id): Path<String>,
Query(params): Query<ProbeQuery>,
) -> Result<Json<serde_json::Value>, ApiError> {
auth_user.require_role(ROLE_USER)?;
let locator = params.locator.trim().to_string();
if locator.is_empty() {
return Err(ApiError::bad_request("locator must not be empty"));
}
// Verify the archive exists but don't need the paths for probing.
let _ = mounted_archive(&state, &archive_id)?;
// Resolve to a yt-dlp URL; return empty result immediately for non-video sources.
let Some(ytdlp_url) = capture::locator_to_ytdlp_url(&locator) else {
return Ok(Json(serde_json::json!({ "has_video": false, "has_audio": false, "qualities": [] })));
};
// fetch_metadata shells out and can take several seconds — keep the async runtime free.
// Returns None when yt-dlp exits non-zero (transient error, rate-limit, unsupported
// extractor, etc.). That is distinct from "yt-dlp ran fine but found no video": we
// return 502 so the frontend treats it as inconclusive rather than showing
// "No video detected" for a URL that may well be downloadable.
let maybe_result = tokio::task::spawn_blocking(move || {
downloader::ytdlp::fetch_metadata(&ytdlp_url)
.map(|json| downloader::ytdlp::probe_result(&json))
})
.await
.map_err(|_| ApiError::internal("probe task panicked"))?;
let result = maybe_result.ok_or_else(|| ApiError {
status: StatusCode::BAD_GATEWAY,
message: "yt-dlp metadata fetch failed".to_string(),
})?;
let qualities: Vec<String> = result.video_heights.iter().map(|h| format!("{h}p")).collect();
Ok(Json(serde_json::json!({
"has_video": !qualities.is_empty(),
"qualities": qualities,
"has_audio": result.has_audio,
})))
}
async fn auth_setup_status( async fn auth_setup_status(
State(state): State<AppState>, State(state): State<AppState>,
) -> Result<Json<serde_json::Value>, ApiError> { ) -> Result<Json<serde_json::Value>, ApiError> {
@ -2206,6 +2286,93 @@ mod tests {
assert_eq!(json["status"], "pending"); assert_eq!(json["status"], "pending");
} }
#[tokio::test]
async fn capture_with_valid_quality_is_accepted() {
let dir = tempfile::tempdir().unwrap();
let (registry, _, auth_path) = make_test_registry(&dir);
let session_cookie = make_test_session(&auth_path);
let response = app(registry, auth_path)
.oneshot(
Request::builder()
.method("POST")
.uri("/api/archives/test/captures")
.header("content-type", "application/json")
.header("cookie", &session_cookie)
.body(Body::from(r#"{"locator":"local:/nonexistent","quality":"720p"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::ACCEPTED);
let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(json["job_uid"].as_str().is_some(), "response must have job_uid");
assert_eq!(json["status"], "pending");
}
#[tokio::test]
async fn capture_with_invalid_quality_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let (registry, _, auth_path) = make_test_registry(&dir);
let session_cookie = make_test_session(&auth_path);
let response = app(registry, auth_path)
.oneshot(
Request::builder()
.method("POST")
.uri("/api/archives/test/captures")
.header("content-type", "application/json")
.header("cookie", &session_cookie)
.body(Body::from(r#"{"locator":"local:/nonexistent","quality":"4K"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(json["error"].as_str().is_some_and(|e| e.contains("invalid quality")));
}
#[tokio::test]
async fn probe_requires_auth() {
let (test_app, _dir) = make_test_app();
let response = test_app
.oneshot(
Request::builder()
.uri("/api/archives/test/captures/probe?locator=local%3A%2Fnonexistent")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn probe_non_video_locator_returns_has_video_false() {
// local:/nonexistent is not a yt-dlp source — the handler returns
// immediately without spawning yt-dlp, so this is fast and deterministic.
let dir = tempfile::tempdir().unwrap();
let (registry, _, auth_path) = make_test_registry(&dir);
let session_cookie = make_test_session(&auth_path);
let response = app(registry, auth_path)
.oneshot(
Request::builder()
.uri("/api/archives/test/captures/probe?locator=local%3A%2Fnonexistent")
.header("cookie", &session_cookie)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["has_video"], false);
assert_eq!(json["qualities"], serde_json::json!([]));
assert_eq!(json["has_audio"], false);
}
#[tokio::test] #[tokio::test]
async fn admin_users_requires_admin_role() { async fn admin_users_requires_admin_role() {
let (test_app, _dir) = make_test_app(); let (test_app, _dir) = make_test_app();

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,8 +4,8 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title> <title>Archivr</title>
<script type="module" crossorigin src="/assets/index-CTwpNgei.js"></script> <script type="module" crossorigin src="/assets/index-BHg5-TAr.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BZfMx3eM.css"> <link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css">
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

View file

@ -146,6 +146,35 @@ Auth and session handling will be designed when remote or public hosting becomes
- X/Twitter Tweet content scrape: [Tweet and Thread shorthands](#supported-shorthand-inputs). (These are saved as JSON files in `raw_tweets/`) - X/Twitter Tweet content scrape: [Tweet and Thread shorthands](#supported-shorthand-inputs). (These are saved as JSON files in `raw_tweets/`)
- Instagram, Facebook, TikTok, Reddit, Snapchat: direct URLs or platform-prefixed shorthand passed through to `yt-dlp` - Instagram, Facebook, TikTok, Reddit, Snapchat: direct URLs or platform-prefixed shorthand passed through to `yt-dlp`
#### Video quality and audio-only downloads
When capturing via the web UI, entering a URL for a yt-dlp-backed source (YouTube, Instagram, TikTok, Facebook, Reddit, Snapchat, X media) triggers a metadata probe via `GET /api/archives/:id/captures/probe`. The quality selector then shows only the heights actually available in that video plus **Best quality** (default). An **Audio only** option is appended whenever the probe confirms an audio track exists. UI behaviour by probe outcome:
| `qualities` | `has_audio` | UI shows |
|---|---|---|
| `["1080p", "720p", …]` | `true` | Best / heights / Audio only |
| `["1080p", …]` | `false` | Best / heights |
| `[]` | `true` | Audio only (pre-selected, no Best option) |
| `[]` | `false` | "No media detected" |
| probe fails (502) | — | picker hidden, capture still submittable |
The `POST /api/archives/:id/captures` endpoint accepts an optional `quality` field: `"best"`, `"audio"`, or any `"NNNp"` height string:
```json
{ "locator": "https://www.youtube.com/watch?v=...", "quality": "720p" }
{ "locator": "https://www.youtube.com/watch?v=...", "quality": "audio" }
```
`"audio"` selects the most efficient native audio track without transcoding: Opus/WebM is preferred (smallest at equivalent quality), then AAC/M4A, then whatever yt-dlp considers best. The saved file's extension matches the native format (`.webm` for Opus, `.m4a` for AAC, etc.) — no ffmpeg re-encode, no size inflation. Any `"NNNp"` height is accepted; the server builds the yt-dlp format selector with an unconditional `/best` fallback so the download succeeds even if the exact height is unavailable. Omitting `quality` or passing `"best"` downloads at the highest available quality. Anything else is rejected with HTTP 400.
The probe endpoint (`GET /api/archives/:id/captures/probe?locator=…`) requires auth and returns 200 with:
```json
{ "has_video": true, "has_audio": true, "qualities": ["1080p", "720p", "480p"] }
{ "has_video": false, "has_audio": true, "qualities": [] }
{ "has_video": false, "has_audio": false, "qualities": [] }
```
`has_video: false, has_audio: false` means yt-dlp found no downloadable tracks (e.g. a tweet with no media). A 502 means yt-dlp itself failed (transient network error, rate-limit, unsupported extractor) — treat as inconclusive, not "no media."
### Hosting on NixOS ### Hosting on NixOS
The flake exposes a `nixosModules.default` output. Add it to your system flake and The flake exposes a `nixosModules.default` output. Add it to your system flake and

View file

@ -95,11 +95,13 @@ export async function fetchTags(archiveId) {
return getJson(`/api/archives/${archiveId}/tags`); return getJson(`/api/archives/${archiveId}/tags`);
} }
export async function submitCapture(archiveId, locator) { export async function submitCapture(archiveId, locator, quality = null) {
const payload = { locator }
if (quality && quality !== 'best') payload.quality = quality
const res = await fetch(`/api/archives/${archiveId}/captures`, { const res = await fetch(`/api/archives/${archiveId}/captures`, {
method: "POST", method: "POST",
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
body: JSON.stringify({ locator }), body: JSON.stringify(payload),
}); });
if (!res.ok) { if (!res.ok) {
const body = await res.json().catch(() => ({})); const body = await res.json().catch(() => ({}));
@ -108,6 +110,12 @@ export async function submitCapture(archiveId, locator) {
return res.json(); // { job_uid, status: "pending" } return res.json(); // { job_uid, status: "pending" }
} }
// Returns { has_video: bool, qualities: string[] } e.g. { has_video: true, qualities: ["1080p","720p","480p"] }
// Throws on network error; returns { has_video: false, qualities: [] } on non-video locators.
export async function probeCapture(archiveId, locator) {
return getJson(`/api/archives/${archiveId}/captures/probe?locator=${encodeURIComponent(locator)}`);
}
export async function pollCaptureJob(archiveId, jobUid) { export async function pollCaptureJob(archiveId, jobUid) {
return getJson(`/api/archives/${archiveId}/capture_jobs/${jobUid}`); return getJson(`/api/archives/${archiveId}/capture_jobs/${jobUid}`);
} }

View file

@ -1,10 +1,65 @@
import { useRef, useEffect, useState } from 'react' import { useRef, useEffect, useState } from 'react'
import { submitCapture, pollCaptureJob } from '../api' import { submitCapture, pollCaptureJob, probeCapture } from '../api'
let nextItemId = 1 let nextItemId = 1
// Returns true only for locators that determine_source() routes to yt-dlp download.
// Mirrors the exact conditions in capture.rs playlist/channel shorthands are excluded
// (they return an "not yet implemented" error), as are tweet/thread shorthands.
function isVideoSource(locator) {
const l = locator.trim()
const ll = l.toLowerCase()
// yt: / youtube: shorthands video/short/shorts only; playlist and channel are unsupported
for (const scheme of ['yt:', 'youtube:']) {
if (ll.startsWith(scheme)) {
const after = ll.slice(scheme.length)
return after.startsWith('video/') || after.startsWith('short/') || after.startsWith('shorts/')
}
}
// x: / twitter: / tweet: shorthands only x:media:ID routes to yt-dlp (Source::X)
for (const scheme of ['x:', 'twitter:', 'tweet:']) {
if (ll.startsWith(scheme)) {
return ll.slice(scheme.length).startsWith('media:')
}
}
// Other platform shorthands all go to yt-dlp
if (ll.startsWith('instagram:') || ll.startsWith('facebook:') ||
ll.startsWith('tiktok:') || ll.startsWith('reddit:') ||
ll.startsWith('snapchat:')) return true
// HTTP/HTTPS URLs match the same regexes and prefix checks as determine_source
if (ll.startsWith('http://') || ll.startsWith('https://')) {
// YouTube video (watch, youtu.be, shorts) not playlist or channel
if (/^https?:\/\/(?:www\.)?(?:youtu\.be\/[0-9A-Za-z_-]+|youtube\.com\/watch\?v=[0-9A-Za-z_-]+|youtube\.com\/shorts\/[0-9A-Za-z_-]+)/.test(l)) return true
// x.com Source::X yt-dlp (note: twitter.com URLs fall through to Source::Url, not yt-dlp)
if (ll.startsWith('https://x.com/') || ll.startsWith('http://x.com/')) return true
// Instagram
if (/^https?:\/\/(?:www\.)?instagram\.com\//.test(ll)) return true
// Facebook + fb.watch
if (/^https?:\/\/(?:www\.)?facebook\.com\//.test(ll) || ll.startsWith('https://fb.watch/') || ll.startsWith('http://fb.watch/')) return true
// TikTok
if (/^https?:\/\/(?:www\.)?tiktok\.com\//.test(ll)) return true
// Reddit + redd.it
if (/^https?:\/\/(?:www\.)?reddit\.com\//.test(ll) || ll.startsWith('https://redd.it/') || ll.startsWith('http://redd.it/')) return true
// Snapchat
if (/^https?:\/\/(?:www\.)?snapchat\.com\//.test(ll)) return true
}
return false
}
function makeItem(locator = '') { function makeItem(locator = '') {
return { id: nextItemId++, locator, status: 'idle', error: null, jobUid: null, archiveId: null } return {
id: nextItemId++, locator, quality: 'best',
// probe: tracks yt-dlp metadata fetch for the locator
probeState: 'idle', // 'idle' | 'probing' | 'done'
probeQualities: null, // null | string[] when done, e.g. ["1080p","720p","480p"]
probeHasAudio: false, // true when probe confirms at least one audio track
status: 'idle', error: null, jobUid: null, archiveId: null,
}
} }
function hasActiveJobs(items) { function hasActiveJobs(items) {
@ -16,6 +71,11 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
const isFirstRenderRef = useRef(true) const isFirstRenderRef = useRef(true)
// jobUid intervalId; survives dialog close since component stays mounted // jobUid intervalId; survives dialog close since component stays mounted
const pollIntervals = useRef(new Map()) const pollIntervals = useRef(new Map())
// itemId debounce timeoutId for probe calls
const probeTimers = useRef(new Map())
// stable ref so probe callbacks always see the current archiveId
const archiveIdRef = useRef(archiveId)
useEffect(() => { archiveIdRef.current = archiveId }, [archiveId])
// Stable refs so polling callbacks always use the latest prop values // Stable refs so polling callbacks always use the latest prop values
const onCapturedRef = useRef(onCaptured) const onCapturedRef = useRef(onCaptured)
@ -63,7 +123,11 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
useEffect(() => { useEffect(() => {
const dialog = dialogRef.current const dialog = dialogRef.current
if (!dialog) return if (!dialog) return
const handler = () => onClose() const handler = () => {
probeTimers.current.forEach(id => clearTimeout(id))
probeTimers.current.clear()
onClose()
}
dialog.addEventListener('close', handler) dialog.addEventListener('close', handler)
return () => dialog.removeEventListener('close', handler) return () => dialog.removeEventListener('close', handler)
}, [onClose]) }, [onClose])
@ -79,13 +143,18 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
isFirstRenderRef.current = false isFirstRenderRef.current = false
if (!dialog.open) dialog.showModal() if (!dialog.open) dialog.showModal()
} else { } else {
probeTimers.current.forEach(id => clearTimeout(id))
probeTimers.current.clear()
if (dialog.open) dialog.close() if (dialog.open) dialog.close()
} }
}, [open]) // eslint-disable-line react-hooks/exhaustive-deps }, [open]) // eslint-disable-line react-hooks/exhaustive-deps
// Clear all intervals on unmount (component teardown) // Clear all intervals and probe timers on unmount (component teardown)
useEffect(() => { useEffect(() => {
return () => pollIntervals.current.forEach(id => clearInterval(id)) return () => {
pollIntervals.current.forEach(id => clearInterval(id))
probeTimers.current.forEach(id => clearTimeout(id))
}
}, []) }, [])
function startPolling(itemId, jobUid, locator, aid) { function startPolling(itemId, jobUid, locator, aid) {
@ -132,9 +201,10 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
if (!item.locator.trim()) return if (!item.locator.trim()) return
const aid = archiveId // capture at submit time const aid = archiveId // capture at submit time
const loc = item.locator.trim() const loc = item.locator.trim()
const qual = item.quality || 'best'
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it)) setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
try { try {
const job = await submitCapture(aid, loc) const job = await submitCapture(aid, loc, qual)
setItems(prev => prev.map(it => setItems(prev => prev.map(it =>
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
)) ))
@ -156,6 +226,8 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
} }
function removeRow(id) { function removeRow(id) {
clearTimeout(probeTimers.current.get(id))
probeTimers.current.delete(id)
setItems(prev => { setItems(prev => {
const next = prev.filter(it => it.id !== id) const next = prev.filter(it => it.id !== id)
return next.length === 0 ? [makeItem()] : next return next.length === 0 ? [makeItem()] : next
@ -169,7 +241,44 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
} }
function updateLocator(id, val) { function updateLocator(id, val) {
setItems(prev => prev.map(it => it.id === id ? { ...it, locator: val } : it)) // Cancel any in-flight debounce and immediately clear stale probe results.
// This prevents old qualities from being visible (and submittable) while
// the 600ms debounce is pending for the new URL.
clearTimeout(probeTimers.current.get(id))
setItems(prev => prev.map(it =>
it.id === id
? { ...it, locator: val, probeState: 'idle', probeQualities: null, probeHasAudio: false, quality: 'best' }
: it
))
if (!isVideoSource(val)) return
// Schedule a fresh probe after the user stops typing
const timer = setTimeout(async () => {
probeTimers.current.delete(id)
setItems(prev => prev.map(it => it.id === id ? { ...it, probeState: 'probing' } : it))
try {
const result = await probeCapture(archiveIdRef.current, val.trim())
setItems(prev => prev.map(it => {
if (it.id !== id || it.locator !== val) return it // stale locator changed again
const qualities = result.qualities ?? []
const hasAudio = result.has_audio ?? false
// Audio-only source: no video heights but audio confirmed force audio mode
const quality = (qualities.length === 0 && hasAudio) ? 'audio' : 'best'
return { ...it, probeState: 'done', probeQualities: qualities, probeHasAudio: hasAudio, quality }
}))
} catch {
// Probe failed (network error, etc.) clear silently; user can still submit
setItems(prev => prev.map(it =>
it.id === id ? { ...it, probeState: 'idle', probeQualities: null } : it
))
}
}, 600)
probeTimers.current.set(id, timer)
}
function updateQuality(id, val) {
setItems(prev => prev.map(it => it.id === id ? { ...it, quality: val } : it))
} }
const pendingCount = items.filter(it => it.status === 'idle' && it.locator.trim()).length const pendingCount = items.filter(it => it.status === 'idle' && it.locator.trim()).length
@ -199,6 +308,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
item={item} item={item}
autoFocus={idx === items.length - 1 && item.status === 'idle'} autoFocus={idx === items.length - 1 && item.status === 'idle'}
onLocatorChange={val => updateLocator(item.id, val)} onLocatorChange={val => updateLocator(item.id, val)}
onQualityChange={val => updateQuality(item.id, val)}
onRemove={() => removeRow(item.id)} onRemove={() => removeRow(item.id)}
onReset={() => resetRow(item.id)} onReset={() => resetRow(item.id)}
onSubmit={handleArchive} onSubmit={handleArchive}
@ -231,7 +341,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
) )
} }
function CaptureRow({ item, autoFocus, onLocatorChange, onRemove, onReset, onSubmit }) { function CaptureRow({ item, autoFocus, onLocatorChange, onQualityChange, onRemove, onReset, onSubmit }) {
const inputRef = useRef(null) const inputRef = useRef(null)
const isActive = item.status === 'submitting' || item.status === 'running' const isActive = item.status === 'submitting' || item.status === 'running'
@ -241,6 +351,49 @@ function CaptureRow({ item, autoFocus, onLocatorChange, onRemove, onReset, onSub
} }
}, [autoFocus]) // eslint-disable-line react-hooks/exhaustive-deps }, [autoFocus]) // eslint-disable-line react-hooks/exhaustive-deps
// Quality control shown right of the input (hidden when active or completed)
const qualityEl = (() => {
if (item.status === 'completed' || isActive) return null
if (!isVideoSource(item.locator)) return null
if (item.probeState === 'probing') {
return <span className="capture-quality-probing" aria-label="Checking available qualities"></span>
}
if (item.probeState === 'done') {
const qualities = item.probeQualities ?? []
const hasAudio = item.probeHasAudio ?? false
if (qualities.length === 0 && !hasAudio) {
return <span className="capture-quality-hint">No media detected</span>
}
if (qualities.length === 0 && hasAudio) {
// Audio-only source: no video tracks, only audio available.
// Don't offer "Best quality" it would request a video format and fail.
return (
<select
className="capture-quality"
value="audio"
onChange={e => onQualityChange(e.target.value)}
aria-label="Video quality"
>
<option value="audio">Audio only</option>
</select>
)
}
return (
<select
className="capture-quality"
value={item.quality || 'best'}
onChange={e => onQualityChange(e.target.value)}
aria-label="Video quality"
>
<option value="best">Best quality</option>
{qualities.map(q => <option key={q} value={q}>{q}</option>)}
{hasAudio && <option value="audio">Audio only</option>}
</select>
)
}
return null // probeState === 'idle', debounce not yet fired
})()
return ( return (
<div className={`capture-row capture-row--${item.status}`}> <div className={`capture-row capture-row--${item.status}`}>
<div className="capture-row-main"> <div className="capture-row-main">
@ -257,6 +410,7 @@ function CaptureRow({ item, autoFocus, onLocatorChange, onRemove, onReset, onSub
autoComplete="off" autoComplete="off"
spellCheck={false} spellCheck={false}
/> />
{qualityEl}
{item.status === 'failed' && ( {item.status === 'failed' && (
<button type="button" className="capture-row-action" onClick={onReset} title="Retry"> <button type="button" className="capture-row-action" onClick={onReset} title="Retry">
<svg viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"> <svg viewBox="0 0 16 16" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">

View file

@ -676,6 +676,39 @@ select {
/* No bottom-margin on inputs inside rows; gap handles spacing */ /* No bottom-margin on inputs inside rows; gap handles spacing */
.capture-row-main .capture-input { margin-bottom: 0; } .capture-row-main .capture-input { margin-bottom: 0; }
/* Quality selector shown for yt-dlp sources */
.capture-quality {
flex-shrink: 0;
height: 32px;
padding: 0 6px;
border: 1px solid var(--line);
border-radius: var(--r);
background: var(--paper);
color: var(--ink);
font-size: 12px;
cursor: pointer;
outline: none;
transition: border-color .15s;
}
.capture-quality:hover { border-color: var(--accent-2); }
.capture-quality:focus { border-color: var(--accent); }
.capture-quality:disabled { opacity: 0.5; cursor: default; }
/* Probing state: ellipsis shown while yt-dlp metadata is in flight */
.capture-quality-probing {
flex-shrink: 0;
font-size: 12px;
color: var(--muted);
padding: 0 4px;
letter-spacing: .05em;
}
/* "No video detected" note */
.capture-quality-hint {
flex-shrink: 0;
font-size: 12px;
color: var(--muted);
font-style: italic;
}
/* Status dot */ /* Status dot */
.cap-dot { .cap-dot {
flex-shrink: 0; flex-shrink: 0;