thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-10-09 12:55:00 +02:00
Code Issues Activity

feat: add YouTube subtitles, local transcription, self-updating yt-dlp/Deno, X Article and thread titles (#39)

Browse source 
- Capture YouTube subtitles by default (opt-out in UI, API, CLI --no-subtitles)
- Summarize YouTube videos from subtitles; fetch on demand, then local transcription, then error
- Local transcription fallback: Whisper, Parakeet, Phonon-2 (English only)
- Runtime-resolved, self-updating yt-dlp and Deno JS runtime (fixes YouTube 403s)
- Settings > Instance > yt-dlp: status and in-app update without restart
- X Article titles from article.title, with idempotent startup backfill
- Thread title generation (single and bulk) with per-provider cheap models
- Per-provider title model settings in Settings > Instance
- Docs, mental model, AGENTS.md and transcription spec updated
This commit is contained in:
TheGeneralist 2026-10-05 19:41:06 +02:00 • committed by GitHub
parent 4f3b2968b6
commit 094f1b0457
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 11377 additions and 583 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/archivr-cli/Cargo.toml
View file

@ -14,8 +14,6 @@ chrono.workspace = true
clap.workspace = true
regex.workspace = true
rusqlite.workspace = true
serde_json.workspace = true
reqwest.workspace = true
[dev-dependencies]
tempfile.workspace = true

325
crates/archivr-cli/src/main.rs
View file

@ -2,25 +2,10 @@ use anyhow::{bail, Context, Result};
use archivr_core::{
archive,
capture::CaptureConfig,
downloader::ytdlp::{
forced_yt_dlp, pinned_yt_dlp, probe_version, resolve_yt_dlp, state_dir, state_dir_yt_dlp,
},
downloader::ytdlp_tools::{tools_status, update_tools, ToolCandidate},
};
use clap::{Parser, Subcommand};
use std::{
env,
path::{Path, PathBuf},
process,
process::Command as ProcCommand,
};
/// GitHub release metadata endpoint for the upstream yt-dlp project.
const YT_DLP_LATEST_RELEASE: &str =
"https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest";
/// Every python zipapp starts with this shebang; used as a sanity check that we
/// downloaded the artifact and not an HTML error page or an LFS pointer.
const ZIPAPP_SHEBANG: &[u8] = b"#!/usr/bin/env python3";
use std::{env, path::Path, process};
#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
@ -35,6 +20,9 @@ enum Command {
Archive {
/// URL or Path to archive
path: String,
/// Skip downloading YouTube subtitles
#[arg(long)]
no_subtitles: bool,
},
Init {
/// Path to initialize the archive in
@ -64,7 +52,7 @@ enum Command {
force_with_info_removal: bool,
},
/// Inspect or update the yt-dlp binary archivr runs
/// Inspect or update the yt-dlp binary and the JavaScript runtime (Deno) archivr runs
#[command(name = "yt-dlp")]
YtDlp {
#[command(subcommand)]
@ -74,13 +62,14 @@ enum Command {
#[derive(Subcommand, Debug)]
enum YtDlpCmd {
/// Download the latest yt-dlp zipapp into archivr's state directory
/// Download the latest yt-dlp zipapp and Deno into archivr's state directory
Update {
/// Install this exact release tag instead of the latest (e.g. 2026.09.15)
/// Install this exact yt-dlp release tag instead of the latest (e.g. 2026.09.15);
/// applies to yt-dlp only — Deno always installs the latest release
#[arg(long)]
version: Option<String>,
},
/// Show every yt-dlp candidate, its version, and which one wins
/// Show every yt-dlp and JS runtime candidate, its version, and which one wins
Status,
}
@ -88,7 +77,7 @@ fn main() -> Result<()> {
let args = Args::parse();
match args.command {
Command::Archive { ref path } => {
Command::Archive { ref path, no_subtitles } => {
let archive_path = match archive::find_archive_path()? {
Some(path) => path,
None => {
@ -97,7 +86,11 @@ fn main() -> Result<()> {
}
};
let archive_paths = archive::read_archive_paths(&archive_path)?;
let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None, &CaptureConfig::default())?;
let config = CaptureConfig {
download_subtitles: !no_subtitles,
..CaptureConfig::default()
};
let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None, &config)?;
println!("Archived: run {}", result.run_uid);
Ok(())
}
@ -139,201 +132,142 @@ fn main() -> Result<()> {
}
}
/// Resolves `<state_dir>/yt-dlp/`, erroring out if there is no usable HOME.
fn yt_dlp_state_dir() -> Result<PathBuf> {
state_dir()
.map(|d| d.join("yt-dlp"))
.context("could not determine a state directory (is $HOME set?)")
}
/// Formats one `status` row. Missing candidates show an em dash.
fn format_status_row(role: &str, path: Option<&Path>, chosen: &Path) -> String {
match path {
Some(p) => {
let version = probe_version(p).unwrap_or_else(|| "—".to_string());
let star = if p == chosen { "*" } else { "" };
format!("{role}\t{}\t{version}\t{star}", p.display())
/// Formats one `status` row: `role\tlocation\tversion\tchosen`. Missing candidates and
/// unknown versions show an em dash.
fn format_status_row(
role: &str,
location: Option<&str>,
version: Option<&str>,
chosen: bool,
) -> String {
match location {
Some(loc) => {
let version = version.unwrap_or("—");
let star = if chosen { "*" } else { "" };
format!("{role}\t{loc}\t{version}\t{star}")
}
None => format!("{role}\t—\t—\t"),
}
}
/// Prints one `status` row. Missing candidates show an em dash.
fn status_row(role: &str, path: Option<&Path>, chosen: &Path) {
println!("{}", format_status_row(role, path, chosen));
/// Renders one candidate row; an invalid override or a candidate whose `--version`
/// probe fails shows its reason in the version column.
fn candidate_row(c: &ToolCandidate) -> String {
match &c.invalid {
Some(reason) => format_status_row(
c.label,
c.path.as_deref(),
Some(&format!("invalid: {reason}")),
c.chosen,
),
None => format_status_row(c.label, c.path.as_deref(), c.version.as_deref(), c.chosen),
}
}
fn yt_dlp_status() -> Result<()> {
let chosen = resolve_yt_dlp();
let s = tools_status();
println!("role\tpath\tversion\tchosen");
status_row(
"force (ARCHIVR_YT_DLP_FORCE)",
forced_yt_dlp().as_deref(),
&chosen,
);
status_row("env (ARCHIVR_YT_DLP)", pinned_yt_dlp().as_deref(), &chosen);
// Show the state-dir slot even when empty, so users can see where an
// `archivr yt-dlp update` would land.
let state_candidate = state_dir_yt_dlp().filter(|p| p.is_file());
status_row("state-dir", state_candidate.as_deref(), &chosen);
status_row(
"path-fallback (yt-dlp)",
Some(Path::new("yt-dlp")),
&chosen,
);
if let Ok(dir) = yt_dlp_state_dir() {
if state_dir_yt_dlp().is_none_or(|p| !p.is_file()) {
println!("\nNo state-dir install yet; `archivr yt-dlp update` would write to {}", dir.join("yt-dlp").display());
}
for c in &s.yt_dlp {
println!("{}", candidate_row(c));
}
if let Some(target) = s.yt_dlp_target.as_deref().filter(|_| !s.yt_dlp_installed) {
println!("\nNo state-dir install yet; `archivr yt-dlp update` would write to {target}");
}
println!("\nJS runtime (passed to yt-dlp as --js-runtimes)");
println!("role\tpath\tversion\tchosen");
for c in &s.js_runtime {
println!("{}", candidate_row(c));
}
if s.js_runtime_chosen.is_none() {
println!(
"\nNo JS runtime resolved — YouTube downloads may fail with HTTP 403; run `archivr yt-dlp update`"
);
}
if let Some(slot) = s.deno_target.as_deref().filter(|_| !s.deno_installed) {
println!("\nNo state-dir deno yet; `archivr yt-dlp update` would write to {slot}");
}
Ok(())
}
/// Asks the GitHub API for the newest yt-dlp release tag.
fn latest_yt_dlp_version(client: &reqwest::blocking::Client) -> Result<String> {
let body = client
.get(YT_DLP_LATEST_RELEASE)
.send()
.context("failed to reach the GitHub releases API")?
.error_for_status()
.context("GitHub releases API returned an error")?
.text()
.context("failed to read the GitHub releases API response")?;
let json: serde_json::Value =
serde_json::from_str(&body).context("GitHub releases API returned invalid JSON")?;
json.get("tag_name")
.and_then(|t| t.as_str())
.map(str::to_string)
.context("GitHub releases API response had no tag_name")
}
/// Installs yt-dlp and Deno independently: a Deno failure never blocks the yt-dlp
/// update (and vice versa). Both outcomes are reported; any failure exits non-zero.
fn yt_dlp_update(requested_version: Option<&str>) -> Result<()> {
let dir = yt_dlp_state_dir()?;
let target = dir.join("yt-dlp");
let staging = dir.join("yt-dlp.new");
let version_file = dir.join(".version");
let report = update_tools(
requested_version,
concat!("archivr-cli/", env!("CARGO_PKG_VERSION")),
false,
&mut |l| println!("{l}"),
)?;
let client = reqwest::blocking::Client::builder()
.user_agent(concat!("archivr-cli/", env!("CARGO_PKG_VERSION")))
.build()
.context("failed to build an HTTP client")?;
let version = match requested_version {
Some(v) => v.to_string(),
None => latest_yt_dlp_version(&client)?,
};
// The sibling .version file is what lets us skip a ~3MB download on a
// no-op update; the binary itself is a zipapp with no cheap version probe
// that doesn't cost a python startup.
let installed = std::fs::read_to_string(&version_file).ok();
if target.is_file() && installed.as_deref().map(str::trim) == Some(version.as_str()) {
println!("yt-dlp {version} is already installed at {}", target.display());
return Ok(());
println!("\nSummary:");
match &report.yt_dlp {
Ok(_) => println!(" yt-dlp: ok"),
Err(e) => println!(" yt-dlp: FAILED: {e:#}"),
}
match &report.deno {
Ok(msg) => println!(" deno: {msg}"),
Err(e) => println!(" deno: FAILED: {e:#}"),
}
println!("Downloading yt-dlp {version}…");
let url = format!("https://github.com/yt-dlp/yt-dlp/releases/download/{version}/yt-dlp");
let bytes = client
.get(&url)
.send()
.with_context(|| format!("failed to download {url}"))?
.error_for_status()
.with_context(|| format!("download failed — is {version} a real release tag?"))?
.bytes()
.context("failed to read the downloaded yt-dlp body")?;
if !bytes.starts_with(ZIPAPP_SHEBANG) {
bail!(
"downloaded artifact from {url} is not a python zipapp \
(expected it to start with `{}`) — refusing to install it",
String::from_utf8_lossy(ZIPAPP_SHEBANG)
);
let failed = report.failed_components();
if !failed.is_empty() {
bail!("update failed for: {}", failed.join(", "));
}
std::fs::create_dir_all(&dir)
.with_context(|| format!("failed to create {}", dir.display()))?;
std::fs::write(&staging, &bytes)
.with_context(|| format!("failed to write {}", staging.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755))
.with_context(|| format!("failed to chmod +x {}", staging.display()))?;
}
// Atomic swap: a concurrently-running archivr sees either the whole old
// binary or the whole new one, never a half-written file.
std::fs::rename(&staging, &target)
.with_context(|| format!("failed to install {}", target.display()))?;
std::fs::write(&version_file, format!("{version}\n"))
.with_context(|| format!("failed to record version in {}", version_file.display()))?;
// The zipapp is python source, not a native binary — installing it on a
// host without python3 is legal (the server may run under a nix wrapper
// with its own PATH) but worth flagging loudly.
let has_python = ProcCommand::new("python3")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false);
if !has_python {
eprintln!(
"warning: python3 was not found on PATH — the yt-dlp zipapp just installed \
at {} will not run until python3 is available",
target.display()
);
}
println!("Installed yt-dlp {version} to {}", target.display());
println!("archivr will now prefer it whenever it is newer than the pinned binary (ARCHIVR_YT_DLP).");
Ok(())
}
#[cfg(test)]
mod tests {
use super::format_status_row;
use super::{candidate_row, format_status_row};
use archivr_core::downloader::ytdlp_tools::ToolCandidate;
use archivr_core::downloader::ytdlp::{
forced_yt_dlp, resolve_yt_dlp_uncached, YT_DLP_FORCE_ENV,
forced_yt_dlp, probe_version, resolve_yt_dlp_uncached, YT_DLP_FORCE_ENV,
};
use std::path::Path;
fn fake_yt_dlp(path: &Path, version: &str) {
/// Writes an executable script to `path` (callers must use a fresh path each time), then
/// waits until it can be exec'd. A child forked by a parallel test while our write fd was
/// open keeps a copy of it until that child execs, so our own exec can fail with ETXTBSY
/// (rust-lang/rust#114554). One exec that isn't ETXTBSY proves no writer is left, and none
/// can appear later because our fd is already closed.
#[cfg(unix)]
fn write_script(path: &Path, body: &str) {
use std::os::unix::fs::PermissionsExt;
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, format!("#!/bin/sh\necho {version}\n")).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap();
std::fs::write(path, body).unwrap();
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap();
for _ in 0..200 {
match std::process::Command::new(path).arg("--version").output() {
Err(e) if e.kind() == std::io::ErrorKind::ExecutableFileBusy => {
std::thread::sleep(std::time::Duration::from_millis(5));
}
_ => return,
}
}
panic!("{} stayed busy (ETXTBSY)", path.display());
}
#[cfg(unix)]
#[test]
fn forced_candidate_is_rendered_and_selected() {
let tmp = tempfile::tempdir().unwrap();
let forced = tmp.path().join("forced/yt-dlp");
fake_yt_dlp(&forced, "2020.01.01");
write_script(&forced, "#!/bin/sh\necho 2020.01.01\n");
unsafe { std::env::set_var(YT_DLP_FORCE_ENV, &forced) };
let candidate = forced_yt_dlp();
assert_eq!(candidate.as_deref(), Some(forced.as_path()));
let chosen = resolve_yt_dlp_uncached();
assert_eq!(chosen, forced);
let version = probe_version(&forced);
assert_eq!(
format_status_row(
"force (ARCHIVR_YT_DLP_FORCE)",
candidate.as_deref(),
&chosen,
Some(&forced.display().to_string()),
version.as_deref(),
chosen == forced,
),
format!(
"force (ARCHIVR_YT_DLP_FORCE)\t{}\t2020.01.01\t*",
@ -343,4 +277,51 @@ mod tests {
unsafe { std::env::remove_var(YT_DLP_FORCE_ENV) };
}
#[test]
fn missing_candidate_renders_dashes() {
assert_eq!(
format_status_row("state-dir", None, Some("2.9.7"), true),
"state-dir\t—\t—\t"
);
}
#[test]
fn unknown_version_renders_dash() {
assert_eq!(
format_status_row("path (deno)", Some("/bin/deno"), None, false),
"path (deno)\t/bin/deno\t—\t"
);
}
#[test]
fn invalid_override_row_shows_reason_and_is_not_chosen() {
assert_eq!(
format_status_row(
"force (ARCHIVR_JS_RUNTIME)",
Some("python"),
Some("invalid: unknown runtime python (expected deno, node, bun or quickjs)"),
false,
),
"force (ARCHIVR_JS_RUNTIME)\tpython\tinvalid: unknown runtime python \
(expected deno, node, bun or quickjs)\t"
);
}
#[test]
fn candidate_row_renders_invalid_override() {
let c = ToolCandidate {
role: "force",
label: "force (ARCHIVR_JS_RUNTIME)",
path: Some("python".into()),
version: None,
chosen: false,
invalid: Some("unknown runtime python (expected deno, node, bun or quickjs)".into()),
};
assert_eq!(
candidate_row(&c),
"force (ARCHIVR_JS_RUNTIME)\tpython\tinvalid: unknown runtime python \
(expected deno, node, bun or quickjs)\t"
);
}
}