1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

Compare commits

...

2 commits

Author SHA1 Message Date
dae61e585d
feat: add user-configurable cookie rules (#20)
Adds per-instance cookie rules (admin-only) that are injected into
every network touchpoint during capture.

Storage:
- New cookie_rules table in the auth DB (idempotent migration)
- Rules have pattern_kind (global/wildcard/regex), optional url_pattern,
  and cookies_json (validated as string-only JSON object)

Matching (resolve_cookies_for_url):
- Global rules always apply
- Wildcard: * and ? with full metacharacter escaping; matched against
  hostname via reqwest::Url when pattern has no ://, full URL otherwise
- Regex: matched against the full URL
- Later rules in ordinal order override earlier ones per cookie name

All six network touchpoints receive resolved cookies:
- http::probe_url_kind and http::download: Cookie request header
- singlefile::save: Netscape cookie file -> --browser-cookies-file
- ytdlp::fetch_metadata and ytdlp::download: Netscape cookie file -> --cookies
- tweets::archive: semicolon credentials file -> --credentials-file
  (only when both ct0 and auth_token are present; otherwise falls back
  to ARCHIVR_TWITTER_CREDENTIALS_FILE)

Security:
- Cookie files written 0o600 (owner read/write only)
- Exact parsed hostname used as cookie domain (no PSL stripping)
- Files deleted unconditionally before any error propagates,
  including spawn failures (hold-result-then-delete pattern)
- No cookie values in process args (no --add-header exposure)

API: GET/POST /api/admin/cookie-rules, PATCH/DELETE /api/admin/cookie-rules/:uid
Frontend: Cookies tab in Settings (admin only) with rule list,
  inline edit, pattern-type selector, client-side JSON validation
CLI: CaptureConfig::default() - no behaviour change

254 tests passing (4 new cookie-rule handler tests)
2026-07-06 19:01:34 +02:00
21b11c211f
feat: YouTube Music audio capture (ytm: shorthand, Spotify detection, stalled job recovery) (#19)
* feat: add YouTube Music and Spotify source detection

- Add Source variants: YouTubeMusicTrack, YouTubeMusicPlaylist,
  SpotifyTrack, SpotifyAlbum, SpotifyPlaylist
- ytm:ID shorthand → music.youtube.com/watch?v=ID (audio-only, forced
  in core regardless of caller quality hint)
- ytm:playlist/ID and music.youtube.com/playlist URLs detected but
  fail with 'not yet implemented' via fail_run
- Spotify URLs/shorthands detected and fail fast with clear DRM error
  via fail_run (after run item created, so status is visible in /runs)
- source_metadata: youtube_music/music/audio and spotify/music/audio
  (entity_kind='music' for UI pill, representation_kind='audio' stored)
- locator_to_ytdlp_url includes YouTubeMusicTrack for probe endpoint
- generate_entry_title: 'Title — Artist' for YTM tracks
- Frontend: isVideoSource handles ytm: and music.youtube.com/watch;
  Spotify returns false (no probe, clear server error on submit)
- Placeholder updated to include ytm:ID
- SOURCE_ICONS: youtube_music (red disc) and spotify (green waves)
- 14 new tests covering all new sources (163 total, all pass)

* fix: prevent yt-dlp playlist expansion and stalled run recovery

- Add --no-playlist to ytdlp::download and fetch_metadata: URLs with a
  list= parameter (e.g. music.youtube.com/watch?v=ID&list=RDAMVM…) no
  longer cause yt-dlp to expand the full playlist and hang; both the
  metadata probe and the download are now single-item only

- Fix fail_stalled_capture_jobs to also recover archive_runs and
  archive_run_items: capture_jobs.run_uid is NULL at crash time so a
  join is unreliable; instead fail all archive_runs/items still
  in_progress directly, then recount failed_count via subquery.
  Startup recovery now makes the Runs UI reflect the correct failed
  state after a hard shutdown

- Expand fail_stalled_jobs_on_restart test to assert archive_run and
  archive_run_item rows are also marked failed, not just capture_jobs

* fix: use play triangle for youtube_music icon
2026-07-06 15:34:14 +02:00
19 changed files with 1371 additions and 106 deletions

1
Cargo.lock generated
View file

@ -132,6 +132,7 @@ dependencies = [
"chrono", "chrono",
"parking_lot", "parking_lot",
"rand", "rand",
"regex",
"rusqlite", "rusqlite",
"serde", "serde",
"serde_json", "serde_json",

View file

@ -1,5 +1,5 @@
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use archivr_core::archive; use archivr_core::{archive, capture::CaptureConfig};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use std::{ use std::{
env, env,
@ -63,7 +63,7 @@ fn main() -> Result<()> {
} }
}; };
let archive_paths = archive::read_archive_paths(&archive_path)?; let archive_paths = archive::read_archive_paths(&archive_path)?;
let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None)?; let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None, &CaptureConfig::default())?;
println!("Archived: run {}", result.run_uid); println!("Archived: run {}", result.run_uid);
Ok(()) Ok(())
} }

View file

@ -3,7 +3,7 @@ use chrono::Local;
use uuid::Uuid; use uuid::Uuid;
use serde_json::json; use serde_json::json;
use std::{ use std::{
collections::HashSet, collections::{HashMap, HashSet},
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
@ -14,6 +14,11 @@ pub enum Source {
YouTubeVideo, YouTubeVideo,
YouTubePlaylist, YouTubePlaylist,
YouTubeChannel, YouTubeChannel,
YouTubeMusicTrack,
YouTubeMusicPlaylist,
SpotifyTrack,
SpotifyAlbum,
SpotifyPlaylist,
X, X,
Tweet, Tweet,
TweetThread, TweetThread,
@ -64,6 +69,78 @@ impl PlatformMetadata {
} }
} }
/// Configuration passed to `perform_capture` to supply per-instance settings
/// that live outside the archive (e.g. cookies stored in the auth DB).
#[derive(Debug, Clone, Default)]
pub struct CaptureConfig {
pub cookie_rules: Vec<database::CookieRule>,
}
/// Resolves which cookies apply to `url` by evaluating all rules in ordinal order.
///
/// Global rules (`url_pattern = None`) always apply.
/// URL-specific rules apply when their pattern matches:
/// - `wildcard`: `*` and `?` glob; matched against the URL hostname when the
/// pattern does not contain `://`, or the full URL when it does.
/// - `regex`: full regex matched against the full URL.
///
/// Later rules in ordinal order override earlier ones for the same cookie name.
pub fn resolve_cookies_for_url(
rules: &[database::CookieRule],
url: &str,
) -> HashMap<String, String> {
let mut result = HashMap::new();
for rule in rules {
let applies = match rule.url_pattern.as_deref() {
None => true,
Some(pattern) => match rule.pattern_kind.as_str() {
"wildcard" => wildcard_matches(pattern, url),
"regex" => regex::Regex::new(pattern)
.is_ok_and(|re| re.is_match(url)),
_ => false,
},
};
if applies {
if let Ok(map) =
serde_json::from_str::<HashMap<String, String>>(&rule.cookies_json)
{
result.extend(map);
}
}
}
result
}
/// Converts a wildcard pattern (`*` = any substring, `?` = any character) to a
/// regex and tests it against the URL.
///
/// If the pattern contains `://` it is matched against the full URL; otherwise
/// only against the hostname (via `reqwest::Url::parse`) so that patterns like
/// `*.youtube.com` work naturally without matching URL path components.
fn wildcard_matches(pattern: &str, url: &str) -> bool {
use reqwest::Url as ReqwestUrl;
let target: String = if pattern.contains("://") {
url.to_string()
} else {
ReqwestUrl::parse(url)
.ok()
.and_then(|u| u.host_str().map(str::to_string))
.unwrap_or_else(|| url.to_string())
};
let mut pat = String::with_capacity(pattern.len() * 2 + 2);
pat.push('^');
for ch in pattern.chars() {
match ch {
'*' => pat.push_str(".*"),
'?' => pat.push('.'),
c if "$.+[]{}()|^\\".contains(c) => { pat.push('\\'); pat.push(c); }
c => pat.push(c),
}
}
pat.push('$');
regex::Regex::new(&pat).is_ok_and(|re| re.is_match(&target))
}
fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String { fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
match source { match source {
Source::YouTubeVideo => meta.title.clone().unwrap_or_else(|| "YouTube Video".to_string()), Source::YouTubeVideo => meta.title.clone().unwrap_or_else(|| "YouTube Video".to_string()),
@ -72,6 +149,19 @@ fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
"Archival of {}", "Archival of {}",
meta.author.as_deref().unwrap_or("Unknown Channel") meta.author.as_deref().unwrap_or("Unknown Channel")
), ),
Source::YouTubeMusicTrack => {
let title = meta.title.as_deref().unwrap_or("Unknown Track");
match meta.author.as_deref() {
Some(a) => format!("{title} \u{2014} {a}"),
None => title.to_string(),
}
}
Source::YouTubeMusicPlaylist => {
meta.title.clone().unwrap_or_else(|| "YouTube Music Playlist".to_string())
}
Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist => {
meta.title.clone().unwrap_or_else(|| "Spotify Content".to_string())
}
Source::X => format!("X Media by {}", meta.author.as_deref().unwrap_or("unknown")), Source::X => format!("X Media by {}", meta.author.as_deref().unwrap_or("unknown")),
Source::Tweet => { Source::Tweet => {
let excerpt = meta.caption_excerpt().unwrap_or_else(|| "Tweet".to_string()); let excerpt = meta.caption_excerpt().unwrap_or_else(|| "Tweet".to_string());
@ -124,6 +214,32 @@ fn expand_shorthand_to_url(path: &str, source: &Source) -> String {
} }
} }
// YouTube Music shorthands: ytm:ID (track) or ytm:playlist/ID
if matches!(source, Source::YouTubeMusicTrack | Source::YouTubeMusicPlaylist) {
if let Some(after) = path.strip_prefix("ytm:") {
if let Some(id) = after.strip_prefix("playlist/") {
return format!("https://music.youtube.com/playlist?list={id}");
}
// bare ytm:ID → track
return format!("https://music.youtube.com/watch?v={after}");
}
}
// Spotify shorthands: spotify:track:ID, spotify:album:ID, spotify:playlist:ID
if matches!(source, Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist) {
if let Some(after) = path.strip_prefix("spotify:") {
if let Some(id) = after.strip_prefix("track:") {
return format!("https://open.spotify.com/track/{id}");
}
if let Some(id) = after.strip_prefix("album:") {
return format!("https://open.spotify.com/album/{id}");
}
if let Some(id) = after.strip_prefix("playlist:") {
return format!("https://open.spotify.com/playlist/{id}");
}
}
}
if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) { if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) {
if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) { if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) {
return format!("https://x.com/i/status/{tweet_id}"); return format!("https://x.com/i/status/{tweet_id}");
@ -187,6 +303,27 @@ fn determine_source(path: &str) -> Source {
} }
} }
// Shorthand scheme: ytm:
if let Some(after_scheme) = path.strip_prefix("ytm:") {
if after_scheme.starts_with("playlist/") {
return Source::YouTubeMusicPlaylist;
}
// Any other suffix is treated as a track ID
return Source::YouTubeMusicTrack;
}
// Shorthand scheme: spotify:
if let Some(after_scheme) = path.strip_prefix("spotify:") {
if after_scheme.starts_with("album:") {
return Source::SpotifyAlbum;
}
if after_scheme.starts_with("playlist:") {
return Source::SpotifyPlaylist;
}
// track: or anything else maps to a track
return Source::SpotifyTrack;
}
// Shorthand schemes: tweet:, x:, or twitter: // Shorthand schemes: tweet:, x:, or twitter:
if let Some(after_scheme) = path if let Some(after_scheme) = path
.strip_prefix("x:") .strip_prefix("x:")
@ -275,6 +412,37 @@ fn determine_source(path: &str) -> Source {
return Source::YouTubeChannel; return Source::YouTubeChannel;
} }
// YouTube Music track URLs: music.youtube.com/watch?v=ID
if path.starts_with("https://music.youtube.com/watch")
|| path.starts_with("http://music.youtube.com/watch")
{
return Source::YouTubeMusicTrack;
}
// YouTube Music playlist URLs: music.youtube.com/playlist?list=ID
if path.starts_with("https://music.youtube.com/playlist")
|| path.starts_with("http://music.youtube.com/playlist")
{
return Source::YouTubeMusicPlaylist;
}
// Spotify URLs: open.spotify.com/{track,album,playlist}/ID
if path.starts_with("https://open.spotify.com/track/")
|| path.starts_with("http://open.spotify.com/track/")
{
return Source::SpotifyTrack;
}
if path.starts_with("https://open.spotify.com/album/")
|| path.starts_with("http://open.spotify.com/album/")
{
return Source::SpotifyAlbum;
}
if path.starts_with("https://open.spotify.com/playlist/")
|| path.starts_with("http://open.spotify.com/playlist/")
{
return Source::SpotifyPlaylist;
}
if path.starts_with("https://x.com/") { if path.starts_with("https://x.com/") {
return Source::X; return Source::X;
} }
@ -341,6 +509,7 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
let source = determine_source(locator); let source = determine_source(locator);
match source { match source {
Source::YouTubeVideo Source::YouTubeVideo
| Source::YouTubeMusicTrack
| Source::X | Source::X
| Source::Instagram | Source::Instagram
| Source::Facebook | Source::Facebook
@ -426,6 +595,11 @@ fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str)
Source::YouTubeVideo => ("youtube", "video", "video"), Source::YouTubeVideo => ("youtube", "video", "video"),
Source::YouTubePlaylist => ("youtube", "playlist", "container"), Source::YouTubePlaylist => ("youtube", "playlist", "container"),
Source::YouTubeChannel => ("youtube", "channel", "container"), Source::YouTubeChannel => ("youtube", "channel", "container"),
Source::YouTubeMusicTrack => ("youtube_music", "music", "audio"),
Source::YouTubeMusicPlaylist => ("youtube_music", "playlist", "container"),
Source::SpotifyTrack => ("spotify", "music", "audio"),
Source::SpotifyAlbum => ("spotify", "album", "container"),
Source::SpotifyPlaylist => ("spotify", "playlist", "container"),
Source::X => ("x", "post", "video"), Source::X => ("x", "post", "video"),
Source::Tweet => ("x", "tweet", "tweet_json"), Source::Tweet => ("x", "tweet", "tweet_json"),
Source::TweetThread => ("x", "tweet_thread", "tweet_json"), Source::TweetThread => ("x", "tweet_thread", "tweet_json"),
@ -684,6 +858,7 @@ pub fn perform_capture(
locator: &str, locator: &str,
archive_id: Option<&str>, archive_id: Option<&str>,
quality: Option<&str>, quality: Option<&str>,
config: &CaptureConfig,
) -> Result<CaptureResult> { ) -> Result<CaptureResult> {
// Append a UUID so parallel captures starting in the same millisecond // Append a UUID so parallel captures starting in the same millisecond
// never collide on the staging directory or file names. // never collide on the staging directory or file names.
@ -699,6 +874,10 @@ pub fn perform_capture(
let mut source = determine_source(locator); let mut source = determine_source(locator);
// Expand shorthands to the canonical URL for cookie matching.
let canonical_url = expand_shorthand_to_url(locator, &source);
let cookies = resolve_cookies_for_url(&config.cookie_rules, &canonical_url);
// Create the run record before probing so every attempt — including // Create the run record before probing so every attempt — including
// probe failures — is visible in /runs with a proper status and error. // probe failures — is visible in /runs with a proper status and error.
let run = database::create_archive_run(&conn, user_id, 1)?; let run = database::create_archive_run(&conn, user_id, 1)?;
@ -706,7 +885,7 @@ pub fn perform_capture(
// For generic http/https URLs, probe Content-Type to decide whether to // For generic http/https URLs, probe Content-Type to decide whether to
// treat the URL as a raw file download or an HTML page for SingleFile. // treat the URL as a raw file download or an HTML page for SingleFile.
if source == Source::Url { if source == Source::Url {
match downloader::http::probe_url_kind(locator) { match downloader::http::probe_url_kind(locator, &cookies) {
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage, Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
Ok(downloader::http::UrlKind::File) => {} Ok(downloader::http::UrlKind::File) => {}
Err(e) => { Err(e) => {
@ -745,9 +924,30 @@ pub fn perform_capture(
)); ));
} }
// Sources: Spotify — not downloadable; Spotify audio is DRM-protected.
if matches!(source, Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist) {
return Err(fail_run(
&conn,
&run,
&item,
"Spotify downloads are not supported: Spotify audio is DRM-protected and cannot \
be downloaded by yt-dlp. Archive the equivalent YouTube Music track instead.",
));
}
// Sources: YouTube Music Playlist — container expansion not yet implemented.
if source == Source::YouTubeMusicPlaylist {
return Err(fail_run(
&conn,
&run,
&item,
"YouTube Music playlist archiving is not yet implemented.",
));
}
// Source: generic HTTP/S file URL // Source: generic HTTP/S file URL
if source == Source::Url { if source == Source::Url {
match downloader::http::download(locator, store_path, &timestamp) { match downloader::http::download(locator, store_path, &timestamp, &cookies) {
Ok((hash, file_extension, title_hint)) => { Ok((hash, file_extension, title_hint)) => {
let temp_file = store_path let temp_file = store_path
.join("temp") .join("temp")
@ -806,7 +1006,7 @@ pub fn perform_capture(
// Source: web page — archive as a self-contained HTML snapshot via single-file-cli // Source: web page — archive as a self-contained HTML snapshot via single-file-cli
if source == Source::WebPage { if source == Source::WebPage {
match downloader::singlefile::save(locator, store_path, &timestamp) { match downloader::singlefile::save(locator, store_path, &timestamp, &cookies) {
Ok(result) => { Ok(result) => {
let file_extension = ".html".to_string(); let file_extension = ".html".to_string();
let temp_html = store_path let temp_html = store_path
@ -960,11 +1160,16 @@ pub fn perform_capture(
} }
}; };
// Tweet shorthands (tweet:ID) don't expand to a URL, so `canonical_url`
// won't match x.com patterns. Always resolve cookies against x.com so
// that wildcard/global rules containing `ct0`+`auth_token` are picked up.
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
match downloader::tweets::archive( match downloader::tweets::archive(
locator, locator,
source == Source::TweetThread, source == Source::TweetThread,
store_path, store_path,
&timestamp, &timestamp,
&tweet_cookies,
) { ) {
Ok(_) => { Ok(_) => {
let tweet_entry = record_tweet_entry( let tweet_entry = record_tweet_entry(
@ -1002,12 +1207,13 @@ pub fn perform_capture(
// because --dump-json is a simulate flag that suppresses the download. // because --dump-json is a simulate flag that suppresses the download.
let ytdlp_metadata_json: Option<String> = match source { let ytdlp_metadata_json: Option<String> = match source {
Source::YouTubeVideo Source::YouTubeVideo
| Source::YouTubeMusicTrack
| Source::X | Source::X
| Source::Instagram | Source::Instagram
| Source::Facebook | Source::Facebook
| Source::TikTok | Source::TikTok
| Source::Reddit | Source::Reddit
| Source::Snapchat => downloader::ytdlp::fetch_metadata(&path), | Source::Snapchat => downloader::ytdlp::fetch_metadata(&path, &cookies),
_ => None, _ => None,
}; };
@ -1030,7 +1236,7 @@ pub fn perform_capture(
| Source::TikTok | Source::TikTok
| Source::Reddit | Source::Reddit
| Source::Snapchat => { | Source::Snapchat => {
match downloader::ytdlp::download(path.clone(), store_path, &timestamp, quality) { match downloader::ytdlp::download(path.clone(), store_path, &timestamp, quality, &cookies) {
Ok(result) => result, Ok(result) => result,
Err(e) => { Err(e) => {
return Err(fail_run( return Err(fail_run(
@ -1042,6 +1248,20 @@ pub fn perform_capture(
} }
} }
} }
Source::YouTubeMusicTrack => {
// Music tracks are always audio-only regardless of the caller's quality hint.
match downloader::ytdlp::download(path.clone(), store_path, &timestamp, Some("audio"), &cookies) {
Ok(result) => result,
Err(e) => {
return Err(fail_run(
&conn,
&run,
&item,
&format!("Failed to download audio: {e}"),
));
}
}
}
Source::Local => { Source::Local => {
match downloader::local::save(path.clone(), store_path, &timestamp) { match downloader::local::save(path.clone(), store_path, &timestamp) {
Ok(h) => (h, local_file_extension(&path)), Ok(h) => (h, local_file_extension(&path)),
@ -1412,6 +1632,142 @@ mod tests {
} }
} }
#[test]
fn test_youtube_music_sources() {
// --- determine_source ---
let cases = [
TestCase {
url: "https://music.youtube.com/watch?v=MntbN1DdEP0",
expected: Source::YouTubeMusicTrack,
},
TestCase {
url: "http://music.youtube.com/watch?v=MntbN1DdEP0",
expected: Source::YouTubeMusicTrack,
},
TestCase {
url: "https://music.youtube.com/playlist?list=PLtest123",
expected: Source::YouTubeMusicPlaylist,
},
TestCase {
url: "ytm:MntbN1DdEP0",
expected: Source::YouTubeMusicTrack,
},
TestCase {
url: "ytm:playlist/PLtest123",
expected: Source::YouTubeMusicPlaylist,
},
];
for case in &cases {
assert_eq!(
determine_source(case.url),
case.expected,
"Failed for URL: {}",
case.url
);
}
// --- expand_shorthand_to_url ---
assert_eq!(
expand_shorthand_to_url("ytm:MntbN1DdEP0", &Source::YouTubeMusicTrack),
"https://music.youtube.com/watch?v=MntbN1DdEP0"
);
assert_eq!(
expand_shorthand_to_url("ytm:playlist/PLtest123", &Source::YouTubeMusicPlaylist),
"https://music.youtube.com/playlist?list=PLtest123"
);
// Full URL passes through unchanged
assert_eq!(
expand_shorthand_to_url(
"https://music.youtube.com/watch?v=MntbN1DdEP0",
&Source::YouTubeMusicTrack
),
"https://music.youtube.com/watch?v=MntbN1DdEP0"
);
// --- locator_to_ytdlp_url ---
assert_eq!(
locator_to_ytdlp_url("ytm:MntbN1DdEP0"),
Some("https://music.youtube.com/watch?v=MntbN1DdEP0".to_string())
);
assert_eq!(
locator_to_ytdlp_url("https://music.youtube.com/watch?v=MntbN1DdEP0"),
Some("https://music.youtube.com/watch?v=MntbN1DdEP0".to_string())
);
// Playlist is not exposed to the probe endpoint
assert_eq!(locator_to_ytdlp_url("ytm:playlist/PLtest123"), None);
// --- source_metadata ---
assert_eq!(
source_metadata(Source::YouTubeMusicTrack),
("youtube_music", "music", "audio")
);
assert_eq!(
source_metadata(Source::YouTubeMusicPlaylist),
("youtube_music", "playlist", "container")
);
}
#[test]
fn test_spotify_sources() {
// --- determine_source ---
let cases = [
TestCase {
url: "https://open.spotify.com/track/4iV5W9uYEdYUVa79Axb7Rh",
expected: Source::SpotifyTrack,
},
TestCase {
url: "https://open.spotify.com/album/1DFixLWuPkv3KT3TnV35m3",
expected: Source::SpotifyAlbum,
},
TestCase {
url: "https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M",
expected: Source::SpotifyPlaylist,
},
TestCase {
url: "spotify:track:4iV5W9uYEdYUVa79Axb7Rh",
expected: Source::SpotifyTrack,
},
TestCase {
url: "spotify:album:1DFixLWuPkv3KT3TnV35m3",
expected: Source::SpotifyAlbum,
},
TestCase {
url: "spotify:playlist:37i9dQZF1DXcBWIGoYBM5M",
expected: Source::SpotifyPlaylist,
},
];
for case in &cases {
assert_eq!(
determine_source(case.url),
case.expected,
"Failed for URL: {}",
case.url
);
}
// --- expand_shorthand_to_url ---
assert_eq!(
expand_shorthand_to_url("spotify:track:4iV5W9uYEdYUVa79Axb7Rh", &Source::SpotifyTrack),
"https://open.spotify.com/track/4iV5W9uYEdYUVa79Axb7Rh"
);
assert_eq!(
expand_shorthand_to_url("spotify:album:1DFixLWuPkv3KT3TnV35m3", &Source::SpotifyAlbum),
"https://open.spotify.com/album/1DFixLWuPkv3KT3TnV35m3"
);
assert_eq!(
expand_shorthand_to_url(
"spotify:playlist:37i9dQZF1DXcBWIGoYBM5M",
&Source::SpotifyPlaylist
),
"https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M"
);
// --- source_metadata ---
assert_eq!(source_metadata(Source::SpotifyTrack), ("spotify", "music", "audio"));
assert_eq!(source_metadata(Source::SpotifyAlbum), ("spotify", "album", "container"));
assert_eq!(source_metadata(Source::SpotifyPlaylist), ("spotify", "playlist", "container"));
}
#[test] #[test]
fn test_x_sources() { fn test_x_sources() {
let x_cases = [ let x_cases = [

View file

@ -138,6 +138,16 @@ pub struct InstanceSettings {
pub default_entry_visibility: u32, pub default_entry_visibility: u32,
} }
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CookieRule {
pub rule_uid: String,
pub url_pattern: Option<String>,
pub pattern_kind: String,
pub cookies_json: String,
pub ordinal: i64,
pub created_at: String,
}
#[derive(Debug, Clone, serde::Serialize)] #[derive(Debug, Clone, serde::Serialize)]
pub struct CollectionRecord { pub struct CollectionRecord {
pub id: i64, pub id: i64,
@ -464,6 +474,16 @@ pub fn initialize_auth_schema(conn: &Connection) -> Result<()> {
created_at TEXT NOT NULL, created_at TEXT NOT NULL,
last_login_at TEXT last_login_at TEXT
); );
CREATE TABLE IF NOT EXISTS cookie_rules (
id INTEGER PRIMARY KEY,
rule_uid TEXT NOT NULL UNIQUE,
url_pattern TEXT,
pattern_kind TEXT NOT NULL DEFAULT 'global',
cookies_json TEXT NOT NULL DEFAULT '{}',
ordinal INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
"#, "#,
)?; )?;
// Add display_name column to users if not present (idempotent migration) // Add display_name column to users if not present (idempotent migration)
@ -731,6 +751,85 @@ pub fn update_instance_settings(conn: &Connection, settings: &InstanceSettings)
Ok(()) Ok(())
} }
pub fn list_cookie_rules(conn: &Connection) -> Result<Vec<CookieRule>> {
let mut stmt = conn.prepare(
"SELECT rule_uid, url_pattern, pattern_kind, cookies_json, ordinal, created_at
FROM cookie_rules ORDER BY ordinal ASC, created_at ASC",
)?;
let records = stmt
.query_map([], |row| {
Ok(CookieRule {
rule_uid: row.get(0)?,
url_pattern: row.get(1)?,
pattern_kind: row.get(2)?,
cookies_json: row.get(3)?,
ordinal: row.get(4)?,
created_at: row.get(5)?,
})
})?
.collect::<Result<Vec<_>, _>>()?;
Ok(records)
}
pub fn create_cookie_rule(
conn: &Connection,
url_pattern: Option<&str>,
pattern_kind: &str,
cookies_json: &str,
) -> Result<CookieRule> {
let rule_uid = format!("cr_{}", Uuid::new_v4().simple());
let now = Utc::now().to_rfc3339();
let ordinal: i64 = conn.query_row(
"SELECT COALESCE(MAX(ordinal), -1) + 1 FROM cookie_rules",
[],
|r| r.get(0),
)?;
conn.execute(
"INSERT INTO cookie_rules (rule_uid, url_pattern, pattern_kind, cookies_json, ordinal, created_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![rule_uid, url_pattern, pattern_kind, cookies_json, ordinal, now],
)?;
Ok(CookieRule {
rule_uid,
url_pattern: url_pattern.map(str::to_string),
pattern_kind: pattern_kind.to_string(),
cookies_json: cookies_json.to_string(),
ordinal,
created_at: now,
})
}
pub fn update_cookie_rule(
conn: &Connection,
rule_uid: &str,
url_pattern: Option<&str>,
pattern_kind: &str,
cookies_json: &str,
ordinal: i64,
) -> Result<()> {
let rows = conn.execute(
"UPDATE cookie_rules
SET url_pattern = ?1, pattern_kind = ?2, cookies_json = ?3, ordinal = ?4
WHERE rule_uid = ?5",
params![url_pattern, pattern_kind, cookies_json, ordinal, rule_uid],
)?;
if rows == 0 {
anyhow::bail!("cookie rule not found: {rule_uid}");
}
Ok(())
}
pub fn delete_cookie_rule(conn: &Connection, rule_uid: &str) -> Result<()> {
let rows = conn.execute(
"DELETE FROM cookie_rules WHERE rule_uid = ?1",
[rule_uid],
)?;
if rows == 0 {
anyhow::bail!("cookie rule not found: {rule_uid}");
}
Ok(())
}
pub fn get_user_password_hash(conn: &Connection, user_id: i64) -> Result<Option<String>> { pub fn get_user_password_hash(conn: &Connection, user_id: i64) -> Result<Option<String>> {
conn.query_row( conn.query_row(
"SELECT password_hash FROM users WHERE id = ?1", "SELECT password_hash FROM users WHERE id = ?1",
@ -1082,10 +1181,40 @@ pub fn get_capture_job(conn: &Connection, job_uid: &str) -> Result<Option<Captur
.map_err(Into::into) .map_err(Into::into)
} }
/// Marks all 'running' capture jobs as 'failed' with a restart message. /// Marks all interrupted capture jobs, runs, and run items as failed.
/// Called at server startup to clean up jobs interrupted by a previous shutdown. /// Called at server startup to recover from a hard shutdown mid-capture.
///
/// `capture_jobs.run_uid` is NULL when the server crashes before `perform_capture`
/// returns, so we cannot join; instead we fail every `archive_runs` row still
/// `in_progress` directly — any run that survived shutdown unfinished was interrupted.
///
/// Returns the number of `capture_jobs` rows updated (used for the startup log).
pub fn fail_stalled_capture_jobs(conn: &Connection) -> Result<usize> { pub fn fail_stalled_capture_jobs(conn: &Connection) -> Result<usize> {
let now = now_timestamp(); let now = now_timestamp();
// 1. Fail in-progress run items.
conn.execute(
"UPDATE archive_run_items
SET status = 'failed', error_text = 'interrupted by server restart'
WHERE status = 'in_progress'",
[],
)?;
// 2. Fail in-progress archive runs; recount failed items from the updated rows.
conn.execute(
"UPDATE archive_runs
SET status = 'failed',
finished_at = ?1,
failed_count = (
SELECT COUNT(*) FROM archive_run_items
WHERE run_id = archive_runs.id AND status = 'failed'
),
error_summary = 'interrupted by server restart'
WHERE status = 'in_progress'",
[now.clone()],
)?;
// 3. Fail running capture jobs (the polling layer).
let n = conn.execute( let n = conn.execute(
"UPDATE capture_jobs SET status = 'failed', "UPDATE capture_jobs SET status = 'failed',
error_text = 'interrupted by server restart', error_text = 'interrupted by server restart',
@ -1093,6 +1222,7 @@ pub fn fail_stalled_capture_jobs(conn: &Connection) -> Result<usize> {
WHERE status = 'running'", WHERE status = 'running'",
[now], [now],
)?; )?;
Ok(n) Ok(n)
} }
@ -2686,13 +2816,40 @@ mod tests {
#[test] #[test]
fn fail_stalled_jobs_on_restart() { fn fail_stalled_jobs_on_restart() {
let conn = conn(); let conn = conn();
// Simulate an in-progress capture_job (run_uid still NULL — common crash case).
let uid = create_capture_job(&conn, "test").unwrap(); let uid = create_capture_job(&conn, "test").unwrap();
update_capture_job_status(&conn, &uid, "running", None, None).unwrap(); update_capture_job_status(&conn, &uid, "running", None, None).unwrap();
// Simulate an in-progress archive_run and item with no associated capture_job
// (covers the case where run_uid was never written back before the crash).
let user_id = ensure_default_user(&conn).unwrap();
let run = create_archive_run(&conn, user_id, 1).unwrap();
create_archive_run_item(&conn, run.id, None, 0, "https://example.com", None, "web", "file").unwrap();
let n = fail_stalled_capture_jobs(&conn).unwrap(); let n = fail_stalled_capture_jobs(&conn).unwrap();
assert_eq!(n, 1); assert_eq!(n, 1); // one capture_job updated
// capture_job is failed
let job = get_capture_job(&conn, &uid).unwrap().unwrap(); let job = get_capture_job(&conn, &uid).unwrap().unwrap();
assert_eq!(job.status, "failed"); assert_eq!(job.status, "failed");
assert!(job.error_text.as_deref().unwrap().contains("interrupted")); assert!(job.error_text.as_deref().unwrap().contains("interrupted"));
// archive_run is failed
let updated_run: String = conn.query_row(
"SELECT status FROM archive_runs WHERE id = ?1",
[run.id],
|r| r.get(0),
).unwrap();
assert_eq!(updated_run, "failed");
// archive_run_item is failed
let item_status: String = conn.query_row(
"SELECT status FROM archive_run_items WHERE run_id = ?1",
[run.id],
|r| r.get(0),
).unwrap();
assert_eq!(item_status, "failed");
} }
fn make_auth_conn_for_mgmt() -> Connection { fn make_auth_conn_for_mgmt() -> Connection {

View file

@ -0,0 +1,66 @@
use std::{collections::HashMap, io::Write, path::Path};
/// Writes a Netscape-format HTTP cookie file suitable for yt-dlp `--cookies` and
/// single-file `--browser-cookies-file`. The file is created with mode 0o600
/// (owner read/write only) to protect cookie secrets.
///
/// `domain` is the exact hostname of the target URL (as returned by
/// `domain_from_url`). No leading dot is added — cookies are scoped to the
/// exact host only, not all subdomains. This is intentionally conservative:
/// `www.youtube.com` cookies will not leak to `music.youtube.com`.
pub fn write_netscape_cookie_file(
cookies: &HashMap<String, String>,
domain: &str,
path: &Path,
) -> std::io::Result<()> {
#[cfg(unix)]
let mut f = {
use std::os::unix::fs::OpenOptionsExt;
std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(path)?
};
#[cfg(not(unix))]
let mut f = std::fs::File::create(path)?;
writeln!(f, "# Netscape HTTP Cookie File")?;
// Exact hostname — no leading dot, no subdomain wildcard.
// The user controls scope via their URL patterns.
let cookie_domain = domain.to_string();
for (name, value) in cookies {
// Fields: domain include_subdomains path secure expiry name value
// include_subdomains is FALSE because there is no leading dot.
writeln!(f, "{cookie_domain}\tFALSE\t/\tFALSE\t0\t{name}\t{value}")?;
}
Ok(())
}
/// Extracts the exact hostname from a URL, suitable for use as the Netscape
/// cookie domain.
///
/// Uses `reqwest::Url::parse` so IPv6 addresses, auth-in-URL, ports, and
/// non-http schemes are handled correctly. Returns an empty string on parse
/// failure or when the URL has no host component.
///
/// Examples:
/// - `"https://www.youtube.com/watch?v=x"` → `"www.youtube.com"`
/// - `"https://music.youtube.com/"` → `"music.youtube.com"`
/// - `"https://twitter.com/"` → `"twitter.com"`
pub fn domain_from_url(url: &str) -> String {
reqwest::Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(str::to_string))
.unwrap_or_default()
}
/// Formats cookies as a single `Cookie:` header value ("name=value; name2=value2").
pub fn cookies_to_header(cookies: &HashMap<String, String>) -> String {
cookies
.iter()
.map(|(k, v)| format!("{k}={v}"))
.collect::<Vec<_>>()
.join("; ")
}

View file

@ -1,6 +1,7 @@
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use std::path::Path; use std::{collections::HashMap, path::Path};
use crate::downloader::cookies::cookies_to_header;
use crate::hash::hash_file; use crate::hash::hash_file;
/// Whether a URL resolves to an HTML document or a downloadable file. /// Whether a URL resolves to an HTML document or a downloadable file.
@ -16,25 +17,33 @@ pub enum UrlKind {
/// Returns `Err` if the probe fails (network error, non-2xx/405 status). /// Returns `Err` if the probe fails (network error, non-2xx/405 status).
/// Redirects (3xx) are followed automatically by reqwest; only the final /// Redirects (3xx) are followed automatically by reqwest; only the final
/// response status is checked. /// response status is checked.
pub fn probe_url_kind(url: &str) -> Result<UrlKind> { pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10)) .redirect(reqwest::redirect::Policy::limited(10))
.user_agent("archivr/0.1") .user_agent("archivr/0.1")
.build() .build()
.context("failed to build HTTP client")?; .context("failed to build HTTP client")?;
let cookie_header = cookies_to_header(cookies);
// Prefer HEAD: no body transfer. // Prefer HEAD: no body transfer.
let head = client let head = {
.head(url) let mut req = client.head(url);
.send() if !cookie_header.is_empty() {
.with_context(|| format!("failed to probe {url}"))?; req = req.header(reqwest::header::COOKIE, &cookie_header);
}
req.send().with_context(|| format!("failed to probe {url}"))?
};
if head.status() == reqwest::StatusCode::METHOD_NOT_ALLOWED { if head.status() == reqwest::StatusCode::METHOD_NOT_ALLOWED {
// Server rejected HEAD — do a GET but only inspect headers. // Server rejected HEAD — do a GET but only inspect headers.
let get = client let get = {
.get(url) let mut req = client.get(url);
.send() if !cookie_header.is_empty() {
.with_context(|| format!("failed to probe {url}"))?; req = req.header(reqwest::header::COOKIE, &cookie_header);
}
req.send().with_context(|| format!("failed to probe {url}"))?
};
if !get.status().is_success() { if !get.status().is_success() {
bail!("HTTP {} probing {url}", get.status()); bail!("HTTP {} probing {url}", get.status());
} }
@ -74,17 +83,21 @@ pub fn probe_url_kind(url: &str) -> Result<UrlKind> {
/// - The request fails or returns a non-2xx status. /// - The request fails or returns a non-2xx status.
/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead). /// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
/// - The body cannot be written to disk. /// - The body cannot be written to disk.
pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String, Option<String>)> { pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10)) .redirect(reqwest::redirect::Policy::limited(10))
.user_agent("archivr/0.1") .user_agent("archivr/0.1")
.build() .build()
.context("failed to build HTTP client")?; .context("failed to build HTTP client")?;
let response = client let cookie_header = cookies_to_header(cookies);
.get(url) let response = {
.send() let mut req = client.get(url);
.with_context(|| format!("failed to fetch {url}"))?; if !cookie_header.is_empty() {
req = req.header(reqwest::header::COOKIE, &cookie_header);
}
req.send().with_context(|| format!("failed to fetch {url}"))?
};
if !response.status().is_success() { if !response.status().is_success() {
bail!("HTTP {} fetching {url}", response.status()); bail!("HTTP {} fetching {url}", response.status());
@ -393,7 +406,7 @@ mod tests {
#[test] #[test]
fn probe_url_kind_fails_on_unreachable_host() { fn probe_url_kind_fails_on_unreachable_host() {
// 127.0.0.1:1 is guaranteed to refuse connections. // 127.0.0.1:1 is guaranteed to refuse connections.
let err = probe_url_kind("http://127.0.0.1:1/").unwrap_err(); let err = probe_url_kind("http://127.0.0.1:1/", &HashMap::new()).unwrap_err();
let msg = format!("{err:#}"); let msg = format!("{err:#}");
assert!( assert!(
msg.contains("probe") || msg.contains("connect") || msg.contains("refused"), msg.contains("probe") || msg.contains("connect") || msg.contains("refused"),

View file

@ -1,3 +1,4 @@
pub mod cookies;
pub mod local; pub mod local;
pub mod store; pub mod store;
pub mod tweets; pub mod tweets;

View file

@ -1,8 +1,9 @@
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use base64::engine::general_purpose::STANDARD as B64; use base64::engine::general_purpose::STANDARD as B64;
use base64::Engine as _; use base64::Engine as _;
use std::{env, io::Read, path::Path, process::Command}; use std::{collections::HashMap, env, io::Read, path::Path, process::Command};
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
use crate::hash::hash_file; use crate::hash::hash_file;
/// Result of archiving a web page with single-file. /// Result of archiving a web page with single-file.
@ -27,11 +28,11 @@ pub struct SaveResult {
/// Reads two env vars: /// Reads two env vars:
/// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`). /// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`).
/// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`). /// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`).
pub fn save(url: &str, store_path: &Path, timestamp: &str) -> Result<SaveResult> { pub fn save(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<SaveResult> {
let single_file = let single_file =
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string()); env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string()); let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string());
save_with(url, store_path, timestamp, &single_file, &chrome) save_with(url, store_path, timestamp, &single_file, &chrome, cookies)
} }
/// Inner implementation; takes binary paths explicitly so tests can inject them /// Inner implementation; takes binary paths explicitly so tests can inject them
@ -42,6 +43,7 @@ fn save_with(
timestamp: &str, timestamp: &str,
single_file: &str, single_file: &str,
chrome: &str, chrome: &str,
cookies: &HashMap<String, String>,
) -> Result<SaveResult> { ) -> Result<SaveResult> {
let temp_dir = store_path.join("temp").join(timestamp); let temp_dir = store_path.join("temp").join(timestamp);
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?; std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
@ -92,8 +94,20 @@ fn save_with(
.collect(); .collect();
let browser_args = format!("[{}]", quoted.join(",")); let browser_args = format!("[{}]", quoted.join(","));
let out = Command::new(single_file) // Write cookie file if cookies are provided.
.arg(url) // Never pass cookie values in process args (ps exposure).
let cookie_file: Option<std::path::PathBuf> = if !cookies.is_empty() {
let cf = temp_dir.join("cookies.txt");
let domain = domain_from_url(url);
write_netscape_cookie_file(cookies, &domain, &cf)
.context("failed to write single-file cookie file")?;
Some(cf)
} else {
None
};
let mut cmd = Command::new(single_file);
cmd.arg(url)
.arg(&out_file) .arg(&out_file)
.arg(format!("--browser-executable-path={chrome}")) .arg(format!("--browser-executable-path={chrome}"))
.arg("--browser-headless") .arg("--browser-headless")
@ -120,9 +134,21 @@ fn save_with(
// Preserve fonts: defaults strip @font-face rules deemed "unused" or // Preserve fonts: defaults strip @font-face rules deemed "unused" or
// "alternative" (unicode-range subsets), losing CDN-served fonts. // "alternative" (unicode-range subsets), losing CDN-served fonts.
.arg("--remove-unused-fonts=false") .arg("--remove-unused-fonts=false")
.arg("--remove-alternative-fonts=false") .arg("--remove-alternative-fonts=false");
if let Some(cf) = &cookie_file {
cmd.arg(format!("--browser-cookies-file={}", cf.display()));
}
let spawn_result = cmd
.output() .output()
.with_context(|| format!("failed to spawn {single_file} process"))?; .with_context(|| format!("failed to spawn {single_file} process"));
// Delete cookie file unconditionally — including on spawn failure —
// so secrets are never left in store/temp when the capture fails.
if let Some(cf) = &cookie_file {
let _ = std::fs::remove_file(cf);
}
let out = spawn_result?;
if !out.status.success() { if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr); let stderr = String::from_utf8_lossy(&out.stderr);
@ -296,6 +322,7 @@ mod tests {
"test-ts", "test-ts",
"/nonexistent/single-file", "/nonexistent/single-file",
"chromium", "chromium",
&HashMap::new(),
); );
let err = result.unwrap_err(); let err = result.unwrap_err();
let msg = format!("{err:#}"); let msg = format!("{err:#}");

View file

@ -68,7 +68,13 @@ fn build_scraper_args(
/// ///
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary /// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`. /// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) -> Result<bool> { pub fn archive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<bool> {
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?; let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
// Output directory for Tweet JSON files. // Output directory for Tweet JSON files.
let output_dir = store_path.join("raw_tweets"); let output_dir = store_path.join("raw_tweets");
@ -93,20 +99,59 @@ pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) ->
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py")); .unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd); let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
let credentials_file = if let Some(credentials_file) = // Credentials: only use cookie rules as Twitter credentials when the resolved
env::var_os("ARCHIVR_TWITTER_CREDENTIALS_FILE") // map contains both `ct0` AND `auth_token` — the two cookies the scraper
{ // requires. A global cookie rule for an unrelated site must not suppress the
absolutize_path_from_cwd(PathBuf::from(credentials_file), &invocation_cwd) // ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
// The temp file is written in the semicolon-delimited format the Python scraper
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
// subprocess returns so secrets are never left on disk on failure.
let temp_creds_path: Option<PathBuf>;
let credentials_file: PathBuf;
let has_twitter_cookies =
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
if has_twitter_cookies {
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
let creds_str = cookies
.iter()
.map(|(k, v)| format!("{k}={v}"))
.collect::<Vec<_>>()
.join(";");
{
use std::io::Write;
#[cfg(unix)]
let mut f = {
use std::os::unix::fs::OpenOptionsExt;
std::fs::OpenOptions::new()
.write(true).create(true).truncate(true).mode(0o600)
.open(&cf)
.context("failed to write twitter credentials file")?
};
#[cfg(not(unix))]
let mut f = std::fs::File::create(&cf)
.context("failed to write twitter credentials file")?;
f.write_all(creds_str.as_bytes())
.context("failed to write twitter credentials file")?;
}
temp_creds_path = Some(cf.clone());
credentials_file = cf;
} else if let Some(env_path) = env::var_os("ARCHIVR_TWITTER_CREDENTIALS_FILE") {
credentials_file = absolutize_path_from_cwd(PathBuf::from(env_path), &invocation_cwd);
temp_creds_path = None;
if !credentials_file.is_file() {
bail!(
"Twitter credentials file not found: {}",
credentials_file.display()
);
}
} else { } else {
bail!( bail!(
"Twitter scraping requires ARCHIVR_TWITTER_CREDENTIALS_FILE to point to a cookies file." "Twitter scraping requires either cookie rules for x.com/twitter.com \
); or ARCHIVR_TWITTER_CREDENTIALS_FILE to be set."
};
if !credentials_file.is_file() {
bail!(
"Twitter credentials file not found: {}",
credentials_file.display()
); );
} }
@ -116,12 +161,21 @@ pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) ->
cmd.arg(arg); cmd.arg(arg);
} }
let output = cmd.output().with_context(|| { // Hold the Result so we can delete the credentials file before propagating
// any error — including spawn failures.
let spawn_result = cmd.output().with_context(|| {
format!( format!(
"Failed to spawn tweet scraper at {}", "Failed to spawn tweet scraper at {}",
scraper_path.display() scraper_path.display()
) )
})?; });
// Remove temp credentials file unconditionally — secrets must not persist.
if let Some(cf) = &temp_creds_path {
let _ = fs::remove_file(cf);
}
let output = spawn_result?;
if !output.status.success() { if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr); let stderr = String::from_utf8_lossy(&output.stderr);
@ -465,7 +519,7 @@ mod tests {
fs::write(&credentials, "ct0=test;auth_token=test").unwrap(); fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials); set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
let archived = archive("tweet:123", false, &store_path, "ts").unwrap(); let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
assert!(!archived); assert!(!archived);
@ -538,7 +592,7 @@ EOF
set_test_env("ARCHIVR_TWEET_SCRAPER", &script); set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh"); set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
let archived = archive("tweet:123", false, &store_path, "ts").unwrap(); let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
let tweet_file = output_dir.join("tweet-123.json"); let tweet_file = output_dir.join("tweet-123.json");
let contents = fs::read_to_string(&tweet_file).unwrap(); let contents = fs::read_to_string(&tweet_file).unwrap();

View file

@ -1,6 +1,13 @@
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use std::{env, path::{Path, PathBuf}, process::Command}; use std::{
collections::HashMap,
env,
path::{Path, PathBuf},
process::Command,
};
use uuid::Uuid;
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
use crate::hash::hash_file; use crate::hash::hash_file;
/// Returns the yt-dlp `-f` format selector for `quality`. /// Returns the yt-dlp `-f` format selector for `quality`.
@ -96,6 +103,7 @@ pub fn download(
store_path: &Path, store_path: &Path,
timestamp: &String, timestamp: &String,
quality: Option<&str>, quality: Option<&str>,
cookies: &HashMap<String, String>,
) -> Result<(String, String)> { ) -> Result<(String, String)> {
println!("Downloading with yt-dlp: {path}"); println!("Downloading with yt-dlp: {path}");
@ -105,11 +113,28 @@ pub fn download(
let temp_dir = store_path.join("temp").join(timestamp); let temp_dir = store_path.join("temp").join(timestamp);
std::fs::create_dir_all(&temp_dir)?; std::fs::create_dir_all(&temp_dir)?;
// Write a restrictive-permissions cookie file if cookies are provided.
// Never pass cookie values in process args (ps exposure).
let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
let cf_path = temp_dir.join("cookies.txt");
let domain = domain_from_url(&path);
write_netscape_cookie_file(cookies, &domain, &cf_path)
.context("failed to write yt-dlp cookie file")?;
Some(cf_path)
} else {
None
};
// %(ext)s lets yt-dlp write the correct extension for the chosen format. // %(ext)s lets yt-dlp write the correct extension for the chosen format.
let out_template = temp_dir.join(format!("{timestamp}.%(ext)s")); let out_template = temp_dir.join(format!("{timestamp}.%(ext)s"));
let mut cmd = Command::new(&ytdlp); let mut cmd = Command::new(&ytdlp);
cmd.arg(&path).arg("-f").arg(quality_format(quality)); cmd.arg(&path)
.arg("-f").arg(quality_format(quality))
// This function is only called for single-item sources; --no-playlist
// prevents yt-dlp from expanding a list= query parameter into a full
// playlist download (e.g. music.youtube.com/watch?v=ID&list=RDAMVM…).
.arg("--no-playlist");
if is_audio { if is_audio {
// -x guarantees audio-only even when /best falls back to a combined // -x guarantees audio-only even when /best falls back to a combined
// A/V format. No --audio-format → native remux only, no re-encode. // A/V format. No --audio-format → native remux only, no re-encode.
@ -118,12 +143,21 @@ pub fn download(
// Force the video container to mp4 so we always have a known extension. // Force the video container to mp4 so we always have a known extension.
cmd.arg("--merge-output-format").arg("mp4"); cmd.arg("--merge-output-format").arg("mp4");
} }
if let Some(cf) = &cookie_file {
cmd.arg("--cookies").arg(cf);
}
let out = cmd let out = cmd
.arg("-o") .arg("-o")
.arg(&out_template) .arg(&out_template)
.output() .output()
.with_context(|| format!("failed to spawn {ytdlp} process"))?; .with_context(|| format!("failed to spawn {ytdlp} process"));
// Remove cookie file immediately regardless of outcome.
if let Some(cf) = &cookie_file {
let _ = std::fs::remove_file(cf);
}
let out = out?;
if !out.status.success() { if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr); let stderr = String::from_utf8_lossy(&out.stderr);
bail!("yt-dlp failed: {stderr}"); bail!("yt-dlp failed: {stderr}");
@ -161,15 +195,39 @@ fn find_downloaded_file(temp_dir: &Path, timestamp: &str) -> Result<PathBuf> {
/// This is a simulate call — it does NOT download any media. /// This is a simulate call — it does NOT download any media.
/// On failure (non-zero exit or no stdout), prints the captured stderr /// On failure (non-zero exit or no stdout), prints the captured stderr
/// to stderr (for debugging) then returns `None` so callers can proceed. /// to stderr (for debugging) then returns `None` so callers can proceed.
pub fn fetch_metadata(path: &str) -> Option<String> { pub fn fetch_metadata(path: &str, cookies: &HashMap<String, String>) -> Option<String> {
let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string()); let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
let out = std::process::Command::new(&ytdlp) // Write a temp cookie file if needed; UUID-named to avoid collisions.
.arg("--dump-json") let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
.arg(path) let domain = domain_from_url(path);
.output() let p = std::env::temp_dir()
.ok()?; .join(format!("archivr-cookies-{}.txt", Uuid::new_v4().simple()));
write_netscape_cookie_file(cookies, &domain, &p).ok()?;
Some(p)
} else {
None
};
let mut cmd = std::process::Command::new(&ytdlp);
cmd.arg("--dump-json")
// Same rationale as download(): only called for single-item sources;
// prevents --dump-json from emitting one JSON object per playlist item
// when the URL contains a list= parameter.
.arg("--no-playlist");
if let Some(cf) = &cookie_file {
cmd.arg("--cookies").arg(cf);
}
cmd.arg(path);
let out = cmd.output().ok();
// Remove cookie file regardless of outcome.
if let Some(cf) = &cookie_file {
let _ = std::fs::remove_file(cf);
}
let out = out?;
if !out.status.success() { if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr); let stderr = String::from_utf8_lossy(&out.stderr);
eprintln!( eprintln!(

View file

@ -20,6 +20,7 @@ base64.workspace = true
serde_json.workspace = true serde_json.workspace = true
rusqlite.workspace = true rusqlite.workspace = true
parking_lot.workspace = true parking_lot.workspace = true
regex.workspace = true
[dev-dependencies] [dev-dependencies]
tempfile.workspace = true tempfile.workspace = true

View file

@ -264,6 +264,14 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/admin/instance-settings", "/api/admin/instance-settings",
get(get_instance_settings_handler).patch(update_instance_settings_handler), get(get_instance_settings_handler).patch(update_instance_settings_handler),
) )
.route(
"/api/admin/cookie-rules",
get(list_cookie_rules_handler).post(create_cookie_rule_handler),
)
.route(
"/api/admin/cookie-rules/:rule_uid",
patch(update_cookie_rule_handler).delete(delete_cookie_rule_handler),
)
.route( .route(
"/api/archives/:archive_id/collections", "/api/archives/:archive_id/collections",
get(list_collections_handler).post(create_collection_handler), get(list_collections_handler).post(create_collection_handler),
@ -679,6 +687,21 @@ struct PatchTagBody {
name: String, name: String,
} }
#[derive(Debug, serde::Deserialize)]
struct CreateCookieRuleBody {
url_pattern: Option<String>,
pattern_kind: String,
cookies_json: String,
}
#[derive(Debug, serde::Deserialize)]
struct UpdateCookieRuleBody {
url_pattern: Option<serde_json::Value>, // null → clear, string → set, absent → keep
pattern_kind: Option<String>,
cookies_json: Option<String>,
ordinal: Option<i64>,
}
async fn capture_handler( async fn capture_handler(
State(state): State<AppState>, State(state): State<AppState>,
auth_user: AuthUser, auth_user: AuthUser,
@ -710,6 +733,15 @@ async fn capture_handler(
let job_uid = database::create_capture_job(&conn, &archive_id)?; let job_uid = database::create_capture_job(&conn, &archive_id)?;
drop(conn); drop(conn);
// Load cookie rules from the auth DB to pass into the capture background task.
let cookie_rules = {
match database::open_auth_db(&state.auth_db_path) {
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
Err(_) => vec![],
}
};
let capture_config = capture::CaptureConfig { cookie_rules };
// Spawn background capture. // Spawn background capture.
let locator = body.locator.trim().to_string(); let locator = body.locator.trim().to_string();
let quality = body.quality.clone(); let quality = body.quality.clone();
@ -725,7 +757,7 @@ async fn capture_handler(
} }
}; };
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok(); database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok();
match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref()) { match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref(), &capture_config) {
Ok(result) => { Ok(result) => {
database::update_capture_job_status( database::update_capture_job_status(
&conn, &conn,
@ -806,8 +838,13 @@ async fn probe_handler(
// extractor, etc.). That is distinct from "yt-dlp ran fine but found no video": we // extractor, etc.). That is distinct from "yt-dlp ran fine but found no video": we
// return 502 so the frontend treats it as inconclusive rather than showing // return 502 so the frontend treats it as inconclusive rather than showing
// "No video detected" for a URL that may well be downloadable. // "No video detected" for a URL that may well be downloadable.
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
Err(_) => vec![],
};
let cookies = capture::resolve_cookies_for_url(&cookie_rules, &ytdlp_url);
let maybe_result = tokio::task::spawn_blocking(move || { let maybe_result = tokio::task::spawn_blocking(move || {
downloader::ytdlp::fetch_metadata(&ytdlp_url) downloader::ytdlp::fetch_metadata(&ytdlp_url, &cookies)
.map(|json| downloader::ytdlp::probe_result(&json)) .map(|json| downloader::ytdlp::probe_result(&json))
}) })
.await .await
@ -998,6 +1035,118 @@ async fn update_instance_settings_handler(
Ok(StatusCode::NO_CONTENT) Ok(StatusCode::NO_CONTENT)
} }
// ── Cookie rules ──────────────────────────────────────────────────────────────
async fn list_cookie_rules_handler(
State(state): State<AppState>,
auth_user: AuthUser,
) -> Result<Json<Vec<database::CookieRule>>, ApiError> {
auth_user.require_role(ROLE_ADMIN)?;
let conn = database::open_auth_db(&state.auth_db_path)?;
Ok(Json(database::list_cookie_rules(&conn)?))
}
async fn create_cookie_rule_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Json(body): Json<CreateCookieRuleBody>,
) -> Result<(StatusCode, Json<database::CookieRule>), ApiError> {
auth_user.require_role(ROLE_ADMIN)?;
if !["global", "wildcard", "regex"].contains(&body.pattern_kind.as_str()) {
return Err(ApiError::bad_request(
"pattern_kind must be 'global', 'wildcard', or 'regex'",
));
}
if serde_json::from_str::<std::collections::HashMap<String, String>>(&body.cookies_json).is_err() {
return Err(ApiError::bad_request(
"cookies_json must be a JSON object whose values are all strings, e.g. {\"name\": \"value\"}",
));
}
if body.pattern_kind != "global"
&& body.url_pattern.as_deref().unwrap_or("").trim().is_empty()
{
return Err(ApiError::bad_request(
"url_pattern is required for non-global rules",
));
}
if body.pattern_kind == "regex" {
if let Some(pat) = &body.url_pattern {
regex::Regex::new(pat)
.map_err(|e| ApiError::bad_request(&format!("invalid regex: {e}")))?;
}
}
let conn = database::open_auth_db(&state.auth_db_path)?;
let url_pattern = body
.url_pattern
.as_deref()
.filter(|s| !s.trim().is_empty());
let rule = database::create_cookie_rule(&conn, url_pattern, &body.pattern_kind, &body.cookies_json)?;
Ok((StatusCode::CREATED, Json(rule)))
}
async fn update_cookie_rule_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path(rule_uid): Path<String>,
Json(body): Json<UpdateCookieRuleBody>,
) -> Result<StatusCode, ApiError> {
auth_user.require_role(ROLE_ADMIN)?;
let conn = database::open_auth_db(&state.auth_db_path)?;
let rules = database::list_cookie_rules(&conn)?;
let existing = rules
.into_iter()
.find(|r| r.rule_uid == rule_uid)
.ok_or_else(|| ApiError::not_found("cookie rule not found"))?;
let pattern_kind = body.pattern_kind.unwrap_or(existing.pattern_kind);
let cookies_json = body.cookies_json.unwrap_or(existing.cookies_json);
let ordinal = body.ordinal.unwrap_or(existing.ordinal);
// url_pattern: null JSON value → clear, string → set, absent (None) → keep existing
let url_pattern: Option<String> = match body.url_pattern {
Some(serde_json::Value::Null) => None,
Some(serde_json::Value::String(s)) if s.trim().is_empty() => None,
Some(serde_json::Value::String(s)) => Some(s),
None => existing.url_pattern,
_ => existing.url_pattern,
};
if !["global", "wildcard", "regex"].contains(&pattern_kind.as_str()) {
return Err(ApiError::bad_request(
"pattern_kind must be 'global', 'wildcard', or 'regex'",
));
}
if serde_json::from_str::<std::collections::HashMap<String, String>>(&cookies_json).is_err() {
return Err(ApiError::bad_request(
"cookies_json must be a JSON object whose values are all strings, e.g. {\"name\": \"value\"}",
));
}
if pattern_kind == "regex" {
if let Some(pat) = &url_pattern {
regex::Regex::new(pat)
.map_err(|e| ApiError::bad_request(&format!("invalid regex: {e}")))?;
}
}
database::update_cookie_rule(
&conn,
&rule_uid,
url_pattern.as_deref(),
&pattern_kind,
&cookies_json,
ordinal,
)?;
Ok(StatusCode::NO_CONTENT)
}
async fn delete_cookie_rule_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path(rule_uid): Path<String>,
) -> Result<StatusCode, ApiError> {
auth_user.require_role(ROLE_ADMIN)?;
let conn = database::open_auth_db(&state.auth_db_path)?;
database::delete_cookie_rule(&conn, &rule_uid)
.map_err(|_| ApiError::not_found("cookie rule not found"))?;
Ok(StatusCode::NO_CONTENT)
}
// ── Blob / orphan cleanup ───────────────────────────────────────────────────── // ── Blob / orphan cleanup ─────────────────────────────────────────────────────
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
@ -2657,6 +2806,122 @@ mod tests {
assert_eq!(response.status(), StatusCode::NO_CONTENT); assert_eq!(response.status(), StatusCode::NO_CONTENT);
} }
#[tokio::test] #[tokio::test]
async fn cookie_rules_require_admin() {
// Non-admin (no session) should get 401.
let (test_app, _dir) = make_test_app();
let response = test_app
.oneshot(Request::builder().uri("/api/admin/cookie-rules").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn cookie_rules_create_list_delete() {
let dir = tempfile::tempdir().unwrap();
let (registry, _, auth_path) = make_test_registry(&dir);
let session = make_test_session(&auth_path);
let app = app(registry, auth_path);
// Create a global rule with valid string-only cookies.
let create_resp = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/admin/cookie-rules")
.header("content-type", "application/json")
.header("cookie", &session)
.body(Body::from(r#"{"pattern_kind":"global","cookies_json":"{\"session\":\"abc\"}"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(create_resp.status(), StatusCode::CREATED);
let body: serde_json::Value = serde_json::from_slice(
&axum::body::to_bytes(create_resp.into_body(), usize::MAX).await.unwrap(),
).unwrap();
let rule_uid = body["rule_uid"].as_str().unwrap().to_string();
assert_eq!(body["pattern_kind"], "global");
// List: should contain the created rule.
let list_resp = app
.clone()
.oneshot(
Request::builder()
.uri("/api/admin/cookie-rules")
.header("cookie", &session)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(list_resp.status(), StatusCode::OK);
let list: serde_json::Value = serde_json::from_slice(
&axum::body::to_bytes(list_resp.into_body(), usize::MAX).await.unwrap(),
).unwrap();
assert_eq!(list.as_array().unwrap().len(), 1);
// Delete.
let del_resp = app
.clone()
.oneshot(
Request::builder()
.method("DELETE")
.uri(format!("/api/admin/cookie-rules/{rule_uid}"))
.header("cookie", &session)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(del_resp.status(), StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn cookie_rules_rejects_non_string_values() {
let dir = tempfile::tempdir().unwrap();
let (registry, _, auth_path) = make_test_registry(&dir);
let session = make_test_session(&auth_path);
// cookies_json with a numeric value must be rejected — core only accepts string values.
let resp = app(registry, auth_path)
.oneshot(
Request::builder()
.method("POST")
.uri("/api/admin/cookie-rules")
.header("content-type", "application/json")
.header("cookie", &session)
.body(Body::from(r#"{"pattern_kind":"global","cookies_json":"{\"session\":123}"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn cookie_rules_rejects_invalid_regex() {
let dir = tempfile::tempdir().unwrap();
let (registry, _, auth_path) = make_test_registry(&dir);
let session = make_test_session(&auth_path);
let resp = app(registry, auth_path)
.oneshot(
Request::builder()
.method("POST")
.uri("/api/admin/cookie-rules")
.header("content-type", "application/json")
.header("cookie", &session)
.body(Body::from(r#"{"pattern_kind":"regex","url_pattern":"[invalid","cookies_json":"{\"x\":\"y\"}"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn security_headers_present_on_success_response() { async fn security_headers_present_on_success_response() {
let (test_app, _dir) = make_test_app(); let (test_app, _dir) = make_test_app();
let response = test_app let response = test_app

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,7 +4,7 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title> <title>Archivr</title>
<script type="module" crossorigin src="/assets/index-bXPw5mQ7.js"></script> <script type="module" crossorigin src="/assets/index-CHDuICqH.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css"> <link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css">
</head> </head>
<body> <body>

View file

@ -365,6 +365,49 @@ export async function deleteOrphanBlobs(archiveId) {
return res.json() return res.json()
} }
// ── Cookie rules ──────────────────────────────────────────────────────────────
export async function listCookieRules() {
return getJson('/api/admin/cookie-rules')
}
export async function createCookieRule(urlPattern, patternKind, cookiesJson) {
const res = await fetch('/api/admin/cookie-rules', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
url_pattern: urlPattern || null,
pattern_kind: patternKind,
cookies_json: cookiesJson,
}),
})
if (!res.ok) {
const body = await res.json().catch(() => ({}))
throw new Error(body.message || `HTTP ${res.status}`)
}
return res.json()
}
export async function updateCookieRule(ruleUid, patch) {
const res = await fetch(`/api/admin/cookie-rules/${ruleUid}`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(patch),
})
if (!res.ok) {
const body = await res.json().catch(() => ({}))
throw new Error(body.message || `HTTP ${res.status}`)
}
}
export async function deleteCookieRule(ruleUid) {
const res = await fetch(`/api/admin/cookie-rules/${ruleUid}`, { method: 'DELETE' })
if (!res.ok) {
const body = await res.json().catch(() => ({}))
throw new Error(body.message || `HTTP ${res.status}`)
}
}
// ── 401 interceptor ─────────────────────────────────────────────────────────── // ── 401 interceptor ───────────────────────────────────────────────────────────
const _origFetch = window.fetch; const _origFetch = window.fetch;
window.fetch = async (...args) => { window.fetch = async (...args) => {

View file

@ -18,6 +18,11 @@ function isVideoSource(locator) {
} }
} }
// ytm: shorthand track only; playlist is not yet implemented
if (ll.startsWith('ytm:')) {
return !ll.slice(4).startsWith('playlist/')
}
// x: / twitter: / tweet: shorthands only x:media:ID routes to yt-dlp (Source::X) // x: / twitter: / tweet: shorthands only x:media:ID routes to yt-dlp (Source::X)
for (const scheme of ['x:', 'twitter:', 'tweet:']) { for (const scheme of ['x:', 'twitter:', 'tweet:']) {
if (ll.startsWith(scheme)) { if (ll.startsWith(scheme)) {
@ -25,6 +30,9 @@ function isVideoSource(locator) {
} }
} }
// spotify: shorthands all will fail with a clear error; no probe needed
if (ll.startsWith('spotify:')) return false
// Other platform shorthands all go to yt-dlp // Other platform shorthands all go to yt-dlp
if (ll.startsWith('instagram:') || ll.startsWith('facebook:') || if (ll.startsWith('instagram:') || ll.startsWith('facebook:') ||
ll.startsWith('tiktok:') || ll.startsWith('reddit:') || ll.startsWith('tiktok:') || ll.startsWith('reddit:') ||
@ -32,6 +40,8 @@ function isVideoSource(locator) {
// HTTP/HTTPS URLs match the same regexes and prefix checks as determine_source // HTTP/HTTPS URLs match the same regexes and prefix checks as determine_source
if (ll.startsWith('http://') || ll.startsWith('https://')) { if (ll.startsWith('http://') || ll.startsWith('https://')) {
// YouTube Music track (watch) before generic YouTube check
if (/^https?:\/\/music\.youtube\.com\/watch/.test(ll)) return true
// YouTube video (watch, youtu.be, shorts) not playlist or channel // YouTube video (watch, youtu.be, shorts) not playlist or channel
if (/^https?:\/\/(?:www\.)?(?:youtu\.be\/[0-9A-Za-z_-]+|youtube\.com\/watch\?v=[0-9A-Za-z_-]+|youtube\.com\/shorts\/[0-9A-Za-z_-]+)/.test(l)) return true if (/^https?:\/\/(?:www\.)?(?:youtu\.be\/[0-9A-Za-z_-]+|youtube\.com\/watch\?v=[0-9A-Za-z_-]+|youtube\.com\/shorts\/[0-9A-Za-z_-]+)/.test(l)) return true
// x.com Source::X yt-dlp (note: twitter.com URLs fall through to Source::Url, not yt-dlp) // x.com Source::X yt-dlp (note: twitter.com URLs fall through to Source::Url, not yt-dlp)
@ -46,6 +56,8 @@ function isVideoSource(locator) {
if (/^https?:\/\/(?:www\.)?reddit\.com\//.test(ll) || ll.startsWith('https://redd.it/') || ll.startsWith('http://redd.it/')) return true if (/^https?:\/\/(?:www\.)?reddit\.com\//.test(ll) || ll.startsWith('https://redd.it/') || ll.startsWith('http://redd.it/')) return true
// Snapchat // Snapchat
if (/^https?:\/\/(?:www\.)?snapchat\.com\//.test(ll)) return true if (/^https?:\/\/(?:www\.)?snapchat\.com\//.test(ll)) return true
// Spotify all will fail with a clear error; no probe needed
if (ll.startsWith('https://open.spotify.com/') || ll.startsWith('http://open.spotify.com/')) return false
} }
return false return false
@ -402,7 +414,7 @@ function CaptureRow({ item, autoFocus, onLocatorChange, onQualityChange, onRemov
ref={inputRef} ref={inputRef}
className="capture-input" className="capture-input"
type="text" type="text"
placeholder="https://… · yt:ID · tweet:ID · x:ID" placeholder="https://… · yt:ID · ytm:ID · tweet:ID · x:ID"
value={item.locator} value={item.locator}
onChange={e => onLocatorChange(e.target.value)} onChange={e => onLocatorChange(e.target.value)}
onKeyDown={e => { if (e.key === 'Enter') onSubmit() }} onKeyDown={e => { if (e.key === 'Enter') onSubmit() }}

View file

@ -5,6 +5,7 @@ import {
listTokens, createToken, deleteToken, listTokens, createToken, deleteToken,
getInstanceSettings, updateInstanceSettings, getInstanceSettings, updateInstanceSettings,
scanOrphanBlobs, deleteOrphanBlobs, scanOrphanBlobs, deleteOrphanBlobs,
listCookieRules, createCookieRule, updateCookieRule, deleteCookieRule,
} from '../api.js' } from '../api.js'
const ROLE_ADMIN = 4 const ROLE_ADMIN = 4
@ -13,8 +14,8 @@ export default function SettingsView({ tab, onTabChange, archiveId }) {
const { currentUser, setCurrentUser } = useContext(AuthContext) ?? {} const { currentUser, setCurrentUser } = useContext(AuthContext) ?? {}
const isAdmin = currentUser && ((currentUser.role_bits & ROLE_ADMIN) !== 0) const isAdmin = currentUser && ((currentUser.role_bits & ROLE_ADMIN) !== 0)
const tabs = ['profile', 'tokens', ...(isAdmin ? ['instance', 'storage'] : [])] const tabs = ['profile', 'tokens', ...(isAdmin ? ['instance', 'cookies', 'storage'] : [])]
const tabLabels = { profile: 'Profile', tokens: 'API Tokens', instance: 'Instance', storage: 'Storage' } const tabLabels = { profile: 'Profile', tokens: 'API Tokens', instance: 'Instance', cookies: 'Cookies', storage: 'Storage' }
return ( return (
<section className="admin-view"> <section className="admin-view">
@ -32,6 +33,7 @@ export default function SettingsView({ tab, onTabChange, archiveId }) {
{tab === 'profile' && <ProfileTab currentUser={currentUser} setCurrentUser={setCurrentUser} />} {tab === 'profile' && <ProfileTab currentUser={currentUser} setCurrentUser={setCurrentUser} />}
{tab === 'tokens' && <TokensTab />} {tab === 'tokens' && <TokensTab />}
{tab === 'instance' && isAdmin && <InstanceTab />} {tab === 'instance' && isAdmin && <InstanceTab />}
{tab === 'cookies' && isAdmin && <CookiesTab />}
{tab === 'storage' && isAdmin && <StorageTab archiveId={archiveId} />} {tab === 'storage' && isAdmin && <StorageTab archiveId={archiveId} />}
</section> </section>
) )
@ -430,3 +432,210 @@ function StorageTab({ archiveId }) {
</div> </div>
) )
} }
function CookiesTab() {
const [rules, setRules] = useState(null)
const [loading, setLoading] = useState(true)
const [error, setError] = useState(null)
// Form state for adding a new rule
const [patternKind, setPatternKind] = useState('global')
const [urlPattern, setUrlPattern] = useState('')
const [cookiesInput, setCookiesInput] = useState('{}')
const [addMsg, setAddMsg] = useState(null)
const [adding, setAdding] = useState(false)
// Inline-edit state: ruleUid { cookiesInput, saving, msg }
const [edits, setEdits] = useState({})
useEffect(() => {
load()
}, [])
async function load() {
setLoading(true); setError(null)
try { setRules(await listCookieRules()) }
catch (e) { setError(e.message) }
finally { setLoading(false) }
}
async function handleAdd(e) {
e.preventDefault()
// Validate JSON
try { JSON.parse(cookiesInput) } catch {
setAddMsg({ ok: false, text: 'cookies must be valid JSON, e.g. {"session": "abc"}' })
return
}
setAdding(true); setAddMsg(null)
try {
await createCookieRule(
patternKind === 'global' ? null : urlPattern.trim(),
patternKind,
cookiesInput,
)
setUrlPattern('')
setCookiesInput('{}')
setPatternKind('global')
setAddMsg({ ok: true, text: 'Rule added.' })
await load()
} catch (err) {
setAddMsg({ ok: false, text: err.message })
} finally {
setAdding(false)
}
}
async function handleDelete(ruleUid) {
try {
await deleteCookieRule(ruleUid)
await load()
} catch (err) {
setError(err.message)
}
}
function startEdit(rule) {
setEdits(prev => ({
...prev,
[rule.rule_uid]: {
cookiesInput: rule.cookies_json,
saving: false,
msg: null,
}
}))
}
function cancelEdit(ruleUid) {
setEdits(prev => { const n = { ...prev }; delete n[ruleUid]; return n })
}
async function saveEdit(rule) {
const edit = edits[rule.rule_uid]
try { JSON.parse(edit.cookiesInput) } catch {
setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], msg: { ok: false, text: 'Invalid JSON' } } }))
return
}
setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], saving: true, msg: null } }))
try {
await updateCookieRule(rule.rule_uid, { cookies_json: edit.cookiesInput })
cancelEdit(rule.rule_uid)
await load()
} catch (err) {
setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], saving: false, msg: { ok: false, text: err.message } } }))
}
}
if (loading) return <div className="muted">Loading&hellip;</div>
if (error) return <div className="form-msg form-msg--err">{error}</div>
return (
<div style={{ maxWidth: 560 }}>
<div className="form-section">
<h2>Cookie Rules</h2>
<p className="muted" style={{ marginBottom: 12 }}>
Cookies are injected into every capture network request (yt-dlp, HTTP downloads, web-page snapshots).
Global rules apply to all URLs; wildcard and regex rules apply only to matching URLs.
</p>
{rules && rules.length > 0 ? (
<table className="data-table" style={{ width: '100%', marginBottom: 16 }}>
<thead>
<tr>
<th>Pattern</th>
<th>Cookies</th>
<th style={{ width: 100 }}>Actions</th>
</tr>
</thead>
<tbody>
{rules.map(rule => {
const edit = edits[rule.rule_uid]
const patternLabel = rule.url_pattern
? <><span className="muted">{rule.pattern_kind}:</span> <code>{rule.url_pattern}</code></>
: <span className="muted">global (all URLs)</span>
return (
<tr key={rule.rule_uid}>
<td>{patternLabel}</td>
<td>
{edit ? (
<>
<textarea
className="field-input"
style={{ fontFamily: 'monospace', fontSize: 12, width: '100%', minHeight: 60 }}
value={edit.cookiesInput}
onChange={ev => setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], cookiesInput: ev.target.value } }))}
/>
{edit.msg && <div className={`form-msg form-msg--${edit.msg.ok ? 'ok' : 'err'}`}>{edit.msg.text}</div>}
<div style={{ display: 'flex', gap: 8, marginTop: 4 }}>
<button className="btn-primary" style={{ fontSize: 12, padding: '2px 8px' }} disabled={edit.saving} onClick={() => saveEdit(rule)}>
{edit.saving ? 'Saving\u2026' : 'Save'}
</button>
<button className="btn-secondary" style={{ fontSize: 12, padding: '2px 8px' }} onClick={() => cancelEdit(rule.rule_uid)}>Cancel</button>
</div>
</>
) : (
<code style={{ fontSize: 12, wordBreak: 'break-all' }}>{rule.cookies_json}</code>
)}
</td>
<td>
{!edit && (
<div style={{ display: 'flex', gap: 6 }}>
<button className="btn-secondary" style={{ fontSize: 12, padding: '2px 8px' }} onClick={() => startEdit(rule)}>Edit</button>
<button className="btn-danger" style={{ fontSize: 12, padding: '2px 8px' }} onClick={() => handleDelete(rule.rule_uid)}>Del</button>
</div>
)}
</td>
</tr>
)
})}
</tbody>
</table>
) : (
<p className="muted" style={{ marginBottom: 16 }}>No cookie rules defined.</p>
)}
<h3 style={{ marginBottom: 8 }}>Add Rule</h3>
<form onSubmit={handleAdd}>
<div className="form-field">
<label className="form-label">Pattern type</label>
<select className="field-input" value={patternKind} onChange={e => setPatternKind(e.target.value)}>
<option value="global">Global (all URLs)</option>
<option value="wildcard">Wildcard (e.g. *.youtube.com)</option>
<option value="regex">Regex (matched against full URL)</option>
</select>
</div>
{patternKind !== 'global' && (
<div className="form-field">
<label className="form-label">
{patternKind === 'wildcard' ? 'URL/hostname pattern' : 'Regex pattern'}
</label>
<input
className="field-input"
type="text"
value={urlPattern}
onChange={e => setUrlPattern(e.target.value)}
placeholder={patternKind === 'wildcard' ? '*.youtube.com or https://example.com/*' : '.*\\.youtube\\.com.*'}
required
/>
</div>
)}
<div className="form-field">
<label className="form-label">Cookies (JSON object)</label>
<textarea
className="field-input"
style={{ fontFamily: 'monospace', fontSize: 13, minHeight: 70 }}
value={cookiesInput}
onChange={e => setCookiesInput(e.target.value)}
placeholder='{"SESSION": "abc123", "token": "xyz"}'
required
/>
</div>
{addMsg && <div className={`form-msg form-msg--${addMsg.ok ? 'ok' : 'err'}`}>{addMsg.text}</div>}
<button className="btn-primary" type="submit" disabled={adding}>
{adding ? 'Adding\u2026' : 'Add Rule'}
</button>
</form>
</div>
</div>
)
}

View file

@ -35,6 +35,8 @@ export function formatTimestamp(value) {
export const SOURCE_ICONS = { export const SOURCE_ICONS = {
youtube: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="#FF0000" d="M23.5 6.2a3 3 0 0 0-2.1-2.1C19.5 3.6 12 3.6 12 3.6s-7.5 0-9.4.5A3 3 0 0 0 .5 6.2C0 8.1 0 12 0 12s0 3.9.5 5.8a3 3 0 0 0 2.1 2.1c1.9.5 9.4.5 9.4.5s7.5 0 9.4-.5a3 3 0 0 0 2.1-2.1C24 15.9 24 12 24 12s0-3.9-.5-5.8z"/><polygon fill="#fff" points="9.6,15.6 15.8,12 9.6,8.4"/></svg>`, youtube: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="#FF0000" d="M23.5 6.2a3 3 0 0 0-2.1-2.1C19.5 3.6 12 3.6 12 3.6s-7.5 0-9.4.5A3 3 0 0 0 .5 6.2C0 8.1 0 12 0 12s0 3.9.5 5.8a3 3 0 0 0 2.1 2.1c1.9.5 9.4.5 9.4.5s7.5 0 9.4-.5a3 3 0 0 0 2.1-2.1C24 15.9 24 12 24 12s0-3.9-.5-5.8z"/><polygon fill="#fff" points="9.6,15.6 15.8,12 9.6,8.4"/></svg>`,
youtube_music: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><circle cx="12" cy="12" r="12" fill="#FF0000"/><polygon fill="#fff" points="9.6,15.6 15.8,12 9.6,8.4"/></svg>`,
spotify: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><circle cx="12" cy="12" r="12" fill="#1DB954"/><path fill="#fff" d="M17.9 10.9c-3.2-1.9-8.5-2.1-11.6-1.1-.5.1-1-.2-1.1-.6-.1-.5.2-1 .6-1.1 3.5-1.1 9.4-.9 13 1.3.4.2.6.8.3 1.2-.2.5-.8.6-1.2.3zm-.1 2.9c-.2.4-.7.5-1.1.3-2.7-1.6-6.7-2.1-9.9-1.1-.4.1-.9-.1-1-.5-.1-.4.1-.9.5-1 3.6-1.1 8-.5 11.1 1.3.4.2.5.7.4 1zm-1.3 2.8c-.2.3-.6.4-.9.2-2.3-1.4-5.2-1.7-8.6-.9-.3.1-.7-.1-.8-.5-.1-.3.1-.7.4-.8 3.7-.9 7-.5 9.6 1 .4.2.5.6.3.9z"/></svg>`,
x: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M18.2 2h3.3l-7.2 8.2L23 22h-6.6l-5.2-6.8L5 22H1.7l7.7-8.8L1 2h6.8l4.7 6.2zm-1.1 18h1.8L6.9 3.9H5z"/></svg>`, x: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M18.2 2h3.3l-7.2 8.2L23 22h-6.6l-5.2-6.8L5 22H1.7l7.7-8.8L1 2h6.8l4.7 6.2zm-1.1 18h1.8L6.9 3.9H5z"/></svg>`,
instagram: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><linearGradient id="ig" x1="0%" y1="100%" x2="100%" y2="0%"><stop offset="0%" stop-color="#f09433"/><stop offset="25%" stop-color="#e6683c"/><stop offset="50%" stop-color="#dc2743"/><stop offset="75%" stop-color="#cc2366"/><stop offset="100%" stop-color="#bc1888"/></linearGradient></defs><rect width="24" height="24" rx="5" fill="url(#ig)"/><rect x="2.5" y="2.5" width="19" height="19" rx="4" fill="none" stroke="#fff" stroke-width="1.5"/><circle cx="12" cy="12" r="4" fill="none" stroke="#fff" stroke-width="1.5"/><circle cx="17.5" cy="6.5" r="1" fill="#fff"/></svg>`, instagram: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><defs><linearGradient id="ig" x1="0%" y1="100%" x2="100%" y2="0%"><stop offset="0%" stop-color="#f09433"/><stop offset="25%" stop-color="#e6683c"/><stop offset="50%" stop-color="#dc2743"/><stop offset="75%" stop-color="#cc2366"/><stop offset="100%" stop-color="#bc1888"/></linearGradient></defs><rect width="24" height="24" rx="5" fill="url(#ig)"/><rect x="2.5" y="2.5" width="19" height="19" rx="4" fill="none" stroke="#fff" stroke-width="1.5"/><circle cx="12" cy="12" r="4" fill="none" stroke="#fff" stroke-width="1.5"/><circle cx="17.5" cy="6.5" r="1" fill="#fff"/></svg>`,
facebook: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><rect width="24" height="24" rx="4" fill="#1877F2"/><path fill="#fff" d="M16 8h-2c-.6 0-1 .4-1 1v2h3l-.4 3H13v8h-3v-8H8v-3h2V9a4 4 0 0 1 4-4h2v3z"/></svg>`, facebook: `<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><rect width="24" height="24" rx="4" fill="#1877F2"/><path fill="#fff" d="M16 8h-2c-.6 0-1 .4-1 1v2h3l-.4 3H13v8h-3v-8H8v-3h2V9a4 4 0 0 1 4-4h2v3z"/></svg>`,