mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat: add user-configurable cookie rules (#20)
Adds per-instance cookie rules (admin-only) that are injected into every network touchpoint during capture. Storage: - New cookie_rules table in the auth DB (idempotent migration) - Rules have pattern_kind (global/wildcard/regex), optional url_pattern, and cookies_json (validated as string-only JSON object) Matching (resolve_cookies_for_url): - Global rules always apply - Wildcard: * and ? with full metacharacter escaping; matched against hostname via reqwest::Url when pattern has no ://, full URL otherwise - Regex: matched against the full URL - Later rules in ordinal order override earlier ones per cookie name All six network touchpoints receive resolved cookies: - http::probe_url_kind and http::download: Cookie request header - singlefile::save: Netscape cookie file -> --browser-cookies-file - ytdlp::fetch_metadata and ytdlp::download: Netscape cookie file -> --cookies - tweets::archive: semicolon credentials file -> --credentials-file (only when both ct0 and auth_token are present; otherwise falls back to ARCHIVR_TWITTER_CREDENTIALS_FILE) Security: - Cookie files written 0o600 (owner read/write only) - Exact parsed hostname used as cookie domain (no PSL stripping) - Files deleted unconditionally before any error propagates, including spawn failures (hold-result-then-delete pattern) - No cookie values in process args (no --add-header exposure) API: GET/POST /api/admin/cookie-rules, PATCH/DELETE /api/admin/cookie-rules/:uid Frontend: Cookies tab in Settings (admin only) with rule list, inline edit, pattern-type selector, client-side JSON validation CLI: CaptureConfig::default() - no behaviour change 254 tests passing (4 new cookie-rule handler tests)
This commit is contained in:
parent
21b11c211f
commit
dae61e585d
17 changed files with 1013 additions and 103 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -132,6 +132,7 @@ dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rand",
|
"rand",
|
||||||
|
"regex",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use archivr_core::archive;
|
use archivr_core::{archive, capture::CaptureConfig};
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use std::{
|
use std::{
|
||||||
env,
|
env,
|
||||||
|
|
@ -63,7 +63,7 @@ fn main() -> Result<()> {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let archive_paths = archive::read_archive_paths(&archive_path)?;
|
let archive_paths = archive::read_archive_paths(&archive_path)?;
|
||||||
let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None)?;
|
let result = archivr_core::capture::perform_capture(&archive_paths, path, None, None, &CaptureConfig::default())?;
|
||||||
println!("Archived: run {}", result.run_uid);
|
println!("Archived: run {}", result.run_uid);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ use chrono::Local;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashSet,
|
collections::{HashMap, HashSet},
|
||||||
fs,
|
fs,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
};
|
};
|
||||||
|
|
@ -69,6 +69,78 @@ impl PlatformMetadata {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Configuration passed to `perform_capture` to supply per-instance settings
|
||||||
|
/// that live outside the archive (e.g. cookies stored in the auth DB).
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct CaptureConfig {
|
||||||
|
pub cookie_rules: Vec<database::CookieRule>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves which cookies apply to `url` by evaluating all rules in ordinal order.
|
||||||
|
///
|
||||||
|
/// Global rules (`url_pattern = None`) always apply.
|
||||||
|
/// URL-specific rules apply when their pattern matches:
|
||||||
|
/// - `wildcard`: `*` and `?` glob; matched against the URL hostname when the
|
||||||
|
/// pattern does not contain `://`, or the full URL when it does.
|
||||||
|
/// - `regex`: full regex matched against the full URL.
|
||||||
|
///
|
||||||
|
/// Later rules in ordinal order override earlier ones for the same cookie name.
|
||||||
|
pub fn resolve_cookies_for_url(
|
||||||
|
rules: &[database::CookieRule],
|
||||||
|
url: &str,
|
||||||
|
) -> HashMap<String, String> {
|
||||||
|
let mut result = HashMap::new();
|
||||||
|
for rule in rules {
|
||||||
|
let applies = match rule.url_pattern.as_deref() {
|
||||||
|
None => true,
|
||||||
|
Some(pattern) => match rule.pattern_kind.as_str() {
|
||||||
|
"wildcard" => wildcard_matches(pattern, url),
|
||||||
|
"regex" => regex::Regex::new(pattern)
|
||||||
|
.is_ok_and(|re| re.is_match(url)),
|
||||||
|
_ => false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if applies {
|
||||||
|
if let Ok(map) =
|
||||||
|
serde_json::from_str::<HashMap<String, String>>(&rule.cookies_json)
|
||||||
|
{
|
||||||
|
result.extend(map);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Converts a wildcard pattern (`*` = any substring, `?` = any character) to a
|
||||||
|
/// regex and tests it against the URL.
|
||||||
|
///
|
||||||
|
/// If the pattern contains `://` it is matched against the full URL; otherwise
|
||||||
|
/// only against the hostname (via `reqwest::Url::parse`) so that patterns like
|
||||||
|
/// `*.youtube.com` work naturally without matching URL path components.
|
||||||
|
fn wildcard_matches(pattern: &str, url: &str) -> bool {
|
||||||
|
use reqwest::Url as ReqwestUrl;
|
||||||
|
let target: String = if pattern.contains("://") {
|
||||||
|
url.to_string()
|
||||||
|
} else {
|
||||||
|
ReqwestUrl::parse(url)
|
||||||
|
.ok()
|
||||||
|
.and_then(|u| u.host_str().map(str::to_string))
|
||||||
|
.unwrap_or_else(|| url.to_string())
|
||||||
|
};
|
||||||
|
let mut pat = String::with_capacity(pattern.len() * 2 + 2);
|
||||||
|
pat.push('^');
|
||||||
|
for ch in pattern.chars() {
|
||||||
|
match ch {
|
||||||
|
'*' => pat.push_str(".*"),
|
||||||
|
'?' => pat.push('.'),
|
||||||
|
c if "$.+[]{}()|^\\".contains(c) => { pat.push('\\'); pat.push(c); }
|
||||||
|
c => pat.push(c),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pat.push('$');
|
||||||
|
regex::Regex::new(&pat).is_ok_and(|re| re.is_match(&target))
|
||||||
|
}
|
||||||
|
|
||||||
fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
|
fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
|
||||||
match source {
|
match source {
|
||||||
Source::YouTubeVideo => meta.title.clone().unwrap_or_else(|| "YouTube Video".to_string()),
|
Source::YouTubeVideo => meta.title.clone().unwrap_or_else(|| "YouTube Video".to_string()),
|
||||||
|
|
@ -786,6 +858,7 @@ pub fn perform_capture(
|
||||||
locator: &str,
|
locator: &str,
|
||||||
archive_id: Option<&str>,
|
archive_id: Option<&str>,
|
||||||
quality: Option<&str>,
|
quality: Option<&str>,
|
||||||
|
config: &CaptureConfig,
|
||||||
) -> Result<CaptureResult> {
|
) -> Result<CaptureResult> {
|
||||||
// Append a UUID so parallel captures starting in the same millisecond
|
// Append a UUID so parallel captures starting in the same millisecond
|
||||||
// never collide on the staging directory or file names.
|
// never collide on the staging directory or file names.
|
||||||
|
|
@ -801,6 +874,10 @@ pub fn perform_capture(
|
||||||
|
|
||||||
let mut source = determine_source(locator);
|
let mut source = determine_source(locator);
|
||||||
|
|
||||||
|
// Expand shorthands to the canonical URL for cookie matching.
|
||||||
|
let canonical_url = expand_shorthand_to_url(locator, &source);
|
||||||
|
let cookies = resolve_cookies_for_url(&config.cookie_rules, &canonical_url);
|
||||||
|
|
||||||
// Create the run record before probing so every attempt — including
|
// Create the run record before probing so every attempt — including
|
||||||
// probe failures — is visible in /runs with a proper status and error.
|
// probe failures — is visible in /runs with a proper status and error.
|
||||||
let run = database::create_archive_run(&conn, user_id, 1)?;
|
let run = database::create_archive_run(&conn, user_id, 1)?;
|
||||||
|
|
@ -808,7 +885,7 @@ pub fn perform_capture(
|
||||||
// For generic http/https URLs, probe Content-Type to decide whether to
|
// For generic http/https URLs, probe Content-Type to decide whether to
|
||||||
// treat the URL as a raw file download or an HTML page for SingleFile.
|
// treat the URL as a raw file download or an HTML page for SingleFile.
|
||||||
if source == Source::Url {
|
if source == Source::Url {
|
||||||
match downloader::http::probe_url_kind(locator) {
|
match downloader::http::probe_url_kind(locator, &cookies) {
|
||||||
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
|
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
|
||||||
Ok(downloader::http::UrlKind::File) => {}
|
Ok(downloader::http::UrlKind::File) => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -870,7 +947,7 @@ pub fn perform_capture(
|
||||||
|
|
||||||
// Source: generic HTTP/S file URL
|
// Source: generic HTTP/S file URL
|
||||||
if source == Source::Url {
|
if source == Source::Url {
|
||||||
match downloader::http::download(locator, store_path, ×tamp) {
|
match downloader::http::download(locator, store_path, ×tamp, &cookies) {
|
||||||
Ok((hash, file_extension, title_hint)) => {
|
Ok((hash, file_extension, title_hint)) => {
|
||||||
let temp_file = store_path
|
let temp_file = store_path
|
||||||
.join("temp")
|
.join("temp")
|
||||||
|
|
@ -929,7 +1006,7 @@ pub fn perform_capture(
|
||||||
|
|
||||||
// Source: web page — archive as a self-contained HTML snapshot via single-file-cli
|
// Source: web page — archive as a self-contained HTML snapshot via single-file-cli
|
||||||
if source == Source::WebPage {
|
if source == Source::WebPage {
|
||||||
match downloader::singlefile::save(locator, store_path, ×tamp) {
|
match downloader::singlefile::save(locator, store_path, ×tamp, &cookies) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
let file_extension = ".html".to_string();
|
let file_extension = ".html".to_string();
|
||||||
let temp_html = store_path
|
let temp_html = store_path
|
||||||
|
|
@ -1083,11 +1160,16 @@ pub fn perform_capture(
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Tweet shorthands (tweet:ID) don't expand to a URL, so `canonical_url`
|
||||||
|
// won't match x.com patterns. Always resolve cookies against x.com so
|
||||||
|
// that wildcard/global rules containing `ct0`+`auth_token` are picked up.
|
||||||
|
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
|
||||||
match downloader::tweets::archive(
|
match downloader::tweets::archive(
|
||||||
locator,
|
locator,
|
||||||
source == Source::TweetThread,
|
source == Source::TweetThread,
|
||||||
store_path,
|
store_path,
|
||||||
×tamp,
|
×tamp,
|
||||||
|
&tweet_cookies,
|
||||||
) {
|
) {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
let tweet_entry = record_tweet_entry(
|
let tweet_entry = record_tweet_entry(
|
||||||
|
|
@ -1131,7 +1213,7 @@ pub fn perform_capture(
|
||||||
| Source::Facebook
|
| Source::Facebook
|
||||||
| Source::TikTok
|
| Source::TikTok
|
||||||
| Source::Reddit
|
| Source::Reddit
|
||||||
| Source::Snapchat => downloader::ytdlp::fetch_metadata(&path),
|
| Source::Snapchat => downloader::ytdlp::fetch_metadata(&path, &cookies),
|
||||||
_ => None,
|
_ => None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -1154,7 +1236,7 @@ pub fn perform_capture(
|
||||||
| Source::TikTok
|
| Source::TikTok
|
||||||
| Source::Reddit
|
| Source::Reddit
|
||||||
| Source::Snapchat => {
|
| Source::Snapchat => {
|
||||||
match downloader::ytdlp::download(path.clone(), store_path, ×tamp, quality) {
|
match downloader::ytdlp::download(path.clone(), store_path, ×tamp, quality, &cookies) {
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return Err(fail_run(
|
return Err(fail_run(
|
||||||
|
|
@ -1168,7 +1250,7 @@ pub fn perform_capture(
|
||||||
}
|
}
|
||||||
Source::YouTubeMusicTrack => {
|
Source::YouTubeMusicTrack => {
|
||||||
// Music tracks are always audio-only regardless of the caller's quality hint.
|
// Music tracks are always audio-only regardless of the caller's quality hint.
|
||||||
match downloader::ytdlp::download(path.clone(), store_path, ×tamp, Some("audio")) {
|
match downloader::ytdlp::download(path.clone(), store_path, ×tamp, Some("audio"), &cookies) {
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return Err(fail_run(
|
return Err(fail_run(
|
||||||
|
|
|
||||||
|
|
@ -138,6 +138,16 @@ pub struct InstanceSettings {
|
||||||
pub default_entry_visibility: u32,
|
pub default_entry_visibility: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct CookieRule {
|
||||||
|
pub rule_uid: String,
|
||||||
|
pub url_pattern: Option<String>,
|
||||||
|
pub pattern_kind: String,
|
||||||
|
pub cookies_json: String,
|
||||||
|
pub ordinal: i64,
|
||||||
|
pub created_at: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, serde::Serialize)]
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
pub struct CollectionRecord {
|
pub struct CollectionRecord {
|
||||||
pub id: i64,
|
pub id: i64,
|
||||||
|
|
@ -464,6 +474,16 @@ pub fn initialize_auth_schema(conn: &Connection) -> Result<()> {
|
||||||
created_at TEXT NOT NULL,
|
created_at TEXT NOT NULL,
|
||||||
last_login_at TEXT
|
last_login_at TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS cookie_rules (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
rule_uid TEXT NOT NULL UNIQUE,
|
||||||
|
url_pattern TEXT,
|
||||||
|
pattern_kind TEXT NOT NULL DEFAULT 'global',
|
||||||
|
cookies_json TEXT NOT NULL DEFAULT '{}',
|
||||||
|
ordinal INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
"#,
|
"#,
|
||||||
)?;
|
)?;
|
||||||
// Add display_name column to users if not present (idempotent migration)
|
// Add display_name column to users if not present (idempotent migration)
|
||||||
|
|
@ -731,6 +751,85 @@ pub fn update_instance_settings(conn: &Connection, settings: &InstanceSettings)
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn list_cookie_rules(conn: &Connection) -> Result<Vec<CookieRule>> {
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT rule_uid, url_pattern, pattern_kind, cookies_json, ordinal, created_at
|
||||||
|
FROM cookie_rules ORDER BY ordinal ASC, created_at ASC",
|
||||||
|
)?;
|
||||||
|
let records = stmt
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok(CookieRule {
|
||||||
|
rule_uid: row.get(0)?,
|
||||||
|
url_pattern: row.get(1)?,
|
||||||
|
pattern_kind: row.get(2)?,
|
||||||
|
cookies_json: row.get(3)?,
|
||||||
|
ordinal: row.get(4)?,
|
||||||
|
created_at: row.get(5)?,
|
||||||
|
})
|
||||||
|
})?
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
Ok(records)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create_cookie_rule(
|
||||||
|
conn: &Connection,
|
||||||
|
url_pattern: Option<&str>,
|
||||||
|
pattern_kind: &str,
|
||||||
|
cookies_json: &str,
|
||||||
|
) -> Result<CookieRule> {
|
||||||
|
let rule_uid = format!("cr_{}", Uuid::new_v4().simple());
|
||||||
|
let now = Utc::now().to_rfc3339();
|
||||||
|
let ordinal: i64 = conn.query_row(
|
||||||
|
"SELECT COALESCE(MAX(ordinal), -1) + 1 FROM cookie_rules",
|
||||||
|
[],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO cookie_rules (rule_uid, url_pattern, pattern_kind, cookies_json, ordinal, created_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
|
params![rule_uid, url_pattern, pattern_kind, cookies_json, ordinal, now],
|
||||||
|
)?;
|
||||||
|
Ok(CookieRule {
|
||||||
|
rule_uid,
|
||||||
|
url_pattern: url_pattern.map(str::to_string),
|
||||||
|
pattern_kind: pattern_kind.to_string(),
|
||||||
|
cookies_json: cookies_json.to_string(),
|
||||||
|
ordinal,
|
||||||
|
created_at: now,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn update_cookie_rule(
|
||||||
|
conn: &Connection,
|
||||||
|
rule_uid: &str,
|
||||||
|
url_pattern: Option<&str>,
|
||||||
|
pattern_kind: &str,
|
||||||
|
cookies_json: &str,
|
||||||
|
ordinal: i64,
|
||||||
|
) -> Result<()> {
|
||||||
|
let rows = conn.execute(
|
||||||
|
"UPDATE cookie_rules
|
||||||
|
SET url_pattern = ?1, pattern_kind = ?2, cookies_json = ?3, ordinal = ?4
|
||||||
|
WHERE rule_uid = ?5",
|
||||||
|
params![url_pattern, pattern_kind, cookies_json, ordinal, rule_uid],
|
||||||
|
)?;
|
||||||
|
if rows == 0 {
|
||||||
|
anyhow::bail!("cookie rule not found: {rule_uid}");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn delete_cookie_rule(conn: &Connection, rule_uid: &str) -> Result<()> {
|
||||||
|
let rows = conn.execute(
|
||||||
|
"DELETE FROM cookie_rules WHERE rule_uid = ?1",
|
||||||
|
[rule_uid],
|
||||||
|
)?;
|
||||||
|
if rows == 0 {
|
||||||
|
anyhow::bail!("cookie rule not found: {rule_uid}");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_user_password_hash(conn: &Connection, user_id: i64) -> Result<Option<String>> {
|
pub fn get_user_password_hash(conn: &Connection, user_id: i64) -> Result<Option<String>> {
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
"SELECT password_hash FROM users WHERE id = ?1",
|
"SELECT password_hash FROM users WHERE id = ?1",
|
||||||
|
|
|
||||||
66
crates/archivr-core/src/downloader/cookies.rs
Normal file
66
crates/archivr-core/src/downloader/cookies.rs
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
use std::{collections::HashMap, io::Write, path::Path};
|
||||||
|
|
||||||
|
/// Writes a Netscape-format HTTP cookie file suitable for yt-dlp `--cookies` and
|
||||||
|
/// single-file `--browser-cookies-file`. The file is created with mode 0o600
|
||||||
|
/// (owner read/write only) to protect cookie secrets.
|
||||||
|
///
|
||||||
|
/// `domain` is the exact hostname of the target URL (as returned by
|
||||||
|
/// `domain_from_url`). No leading dot is added — cookies are scoped to the
|
||||||
|
/// exact host only, not all subdomains. This is intentionally conservative:
|
||||||
|
/// `www.youtube.com` cookies will not leak to `music.youtube.com`.
|
||||||
|
pub fn write_netscape_cookie_file(
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
|
domain: &str,
|
||||||
|
path: &Path,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
#[cfg(unix)]
|
||||||
|
let mut f = {
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create(true)
|
||||||
|
.truncate(true)
|
||||||
|
.mode(0o600)
|
||||||
|
.open(path)?
|
||||||
|
};
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
let mut f = std::fs::File::create(path)?;
|
||||||
|
|
||||||
|
writeln!(f, "# Netscape HTTP Cookie File")?;
|
||||||
|
// Exact hostname — no leading dot, no subdomain wildcard.
|
||||||
|
// The user controls scope via their URL patterns.
|
||||||
|
let cookie_domain = domain.to_string();
|
||||||
|
for (name, value) in cookies {
|
||||||
|
// Fields: domain include_subdomains path secure expiry name value
|
||||||
|
// include_subdomains is FALSE because there is no leading dot.
|
||||||
|
writeln!(f, "{cookie_domain}\tFALSE\t/\tFALSE\t0\t{name}\t{value}")?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts the exact hostname from a URL, suitable for use as the Netscape
|
||||||
|
/// cookie domain.
|
||||||
|
///
|
||||||
|
/// Uses `reqwest::Url::parse` so IPv6 addresses, auth-in-URL, ports, and
|
||||||
|
/// non-http schemes are handled correctly. Returns an empty string on parse
|
||||||
|
/// failure or when the URL has no host component.
|
||||||
|
///
|
||||||
|
/// Examples:
|
||||||
|
/// - `"https://www.youtube.com/watch?v=x"` → `"www.youtube.com"`
|
||||||
|
/// - `"https://music.youtube.com/"` → `"music.youtube.com"`
|
||||||
|
/// - `"https://twitter.com/"` → `"twitter.com"`
|
||||||
|
pub fn domain_from_url(url: &str) -> String {
|
||||||
|
reqwest::Url::parse(url)
|
||||||
|
.ok()
|
||||||
|
.and_then(|u| u.host_str().map(str::to_string))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Formats cookies as a single `Cookie:` header value ("name=value; name2=value2").
|
||||||
|
pub fn cookies_to_header(cookies: &HashMap<String, String>) -> String {
|
||||||
|
cookies
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| format!("{k}={v}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("; ")
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use std::path::Path;
|
use std::{collections::HashMap, path::Path};
|
||||||
|
|
||||||
|
use crate::downloader::cookies::cookies_to_header;
|
||||||
use crate::hash::hash_file;
|
use crate::hash::hash_file;
|
||||||
|
|
||||||
/// Whether a URL resolves to an HTML document or a downloadable file.
|
/// Whether a URL resolves to an HTML document or a downloadable file.
|
||||||
|
|
@ -16,25 +17,33 @@ pub enum UrlKind {
|
||||||
/// Returns `Err` if the probe fails (network error, non-2xx/405 status).
|
/// Returns `Err` if the probe fails (network error, non-2xx/405 status).
|
||||||
/// Redirects (3xx) are followed automatically by reqwest; only the final
|
/// Redirects (3xx) are followed automatically by reqwest; only the final
|
||||||
/// response status is checked.
|
/// response status is checked.
|
||||||
pub fn probe_url_kind(url: &str) -> Result<UrlKind> {
|
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder()
|
||||||
.redirect(reqwest::redirect::Policy::limited(10))
|
.redirect(reqwest::redirect::Policy::limited(10))
|
||||||
.user_agent("archivr/0.1")
|
.user_agent("archivr/0.1")
|
||||||
.build()
|
.build()
|
||||||
.context("failed to build HTTP client")?;
|
.context("failed to build HTTP client")?;
|
||||||
|
|
||||||
|
let cookie_header = cookies_to_header(cookies);
|
||||||
|
|
||||||
// Prefer HEAD: no body transfer.
|
// Prefer HEAD: no body transfer.
|
||||||
let head = client
|
let head = {
|
||||||
.head(url)
|
let mut req = client.head(url);
|
||||||
.send()
|
if !cookie_header.is_empty() {
|
||||||
.with_context(|| format!("failed to probe {url}"))?;
|
req = req.header(reqwest::header::COOKIE, &cookie_header);
|
||||||
|
}
|
||||||
|
req.send().with_context(|| format!("failed to probe {url}"))?
|
||||||
|
};
|
||||||
|
|
||||||
if head.status() == reqwest::StatusCode::METHOD_NOT_ALLOWED {
|
if head.status() == reqwest::StatusCode::METHOD_NOT_ALLOWED {
|
||||||
// Server rejected HEAD — do a GET but only inspect headers.
|
// Server rejected HEAD — do a GET but only inspect headers.
|
||||||
let get = client
|
let get = {
|
||||||
.get(url)
|
let mut req = client.get(url);
|
||||||
.send()
|
if !cookie_header.is_empty() {
|
||||||
.with_context(|| format!("failed to probe {url}"))?;
|
req = req.header(reqwest::header::COOKIE, &cookie_header);
|
||||||
|
}
|
||||||
|
req.send().with_context(|| format!("failed to probe {url}"))?
|
||||||
|
};
|
||||||
if !get.status().is_success() {
|
if !get.status().is_success() {
|
||||||
bail!("HTTP {} probing {url}", get.status());
|
bail!("HTTP {} probing {url}", get.status());
|
||||||
}
|
}
|
||||||
|
|
@ -74,17 +83,21 @@ pub fn probe_url_kind(url: &str) -> Result<UrlKind> {
|
||||||
/// - The request fails or returns a non-2xx status.
|
/// - The request fails or returns a non-2xx status.
|
||||||
/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
|
/// - The response Content-Type is `text/html` (caller should use a web-page archiver instead).
|
||||||
/// - The body cannot be written to disk.
|
/// - The body cannot be written to disk.
|
||||||
pub fn download(url: &str, store_path: &Path, timestamp: &str) -> Result<(String, String, Option<String>)> {
|
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder()
|
||||||
.redirect(reqwest::redirect::Policy::limited(10))
|
.redirect(reqwest::redirect::Policy::limited(10))
|
||||||
.user_agent("archivr/0.1")
|
.user_agent("archivr/0.1")
|
||||||
.build()
|
.build()
|
||||||
.context("failed to build HTTP client")?;
|
.context("failed to build HTTP client")?;
|
||||||
|
|
||||||
let response = client
|
let cookie_header = cookies_to_header(cookies);
|
||||||
.get(url)
|
let response = {
|
||||||
.send()
|
let mut req = client.get(url);
|
||||||
.with_context(|| format!("failed to fetch {url}"))?;
|
if !cookie_header.is_empty() {
|
||||||
|
req = req.header(reqwest::header::COOKIE, &cookie_header);
|
||||||
|
}
|
||||||
|
req.send().with_context(|| format!("failed to fetch {url}"))?
|
||||||
|
};
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
bail!("HTTP {} fetching {url}", response.status());
|
bail!("HTTP {} fetching {url}", response.status());
|
||||||
|
|
@ -393,7 +406,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn probe_url_kind_fails_on_unreachable_host() {
|
fn probe_url_kind_fails_on_unreachable_host() {
|
||||||
// 127.0.0.1:1 is guaranteed to refuse connections.
|
// 127.0.0.1:1 is guaranteed to refuse connections.
|
||||||
let err = probe_url_kind("http://127.0.0.1:1/").unwrap_err();
|
let err = probe_url_kind("http://127.0.0.1:1/", &HashMap::new()).unwrap_err();
|
||||||
let msg = format!("{err:#}");
|
let msg = format!("{err:#}");
|
||||||
assert!(
|
assert!(
|
||||||
msg.contains("probe") || msg.contains("connect") || msg.contains("refused"),
|
msg.contains("probe") || msg.contains("connect") || msg.contains("refused"),
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
pub mod cookies;
|
||||||
pub mod local;
|
pub mod local;
|
||||||
pub mod store;
|
pub mod store;
|
||||||
pub mod tweets;
|
pub mod tweets;
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use base64::engine::general_purpose::STANDARD as B64;
|
use base64::engine::general_purpose::STANDARD as B64;
|
||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use std::{env, io::Read, path::Path, process::Command};
|
use std::{collections::HashMap, env, io::Read, path::Path, process::Command};
|
||||||
|
|
||||||
|
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
|
||||||
use crate::hash::hash_file;
|
use crate::hash::hash_file;
|
||||||
|
|
||||||
/// Result of archiving a web page with single-file.
|
/// Result of archiving a web page with single-file.
|
||||||
|
|
@ -27,11 +28,11 @@ pub struct SaveResult {
|
||||||
/// Reads two env vars:
|
/// Reads two env vars:
|
||||||
/// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`).
|
/// - `ARCHIVR_SINGLE_FILE`: path to the `single-file` binary (default: `"single-file"`).
|
||||||
/// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`).
|
/// - `ARCHIVR_CHROME`: path to the Chromium/Chrome binary (default: `"chromium"`).
|
||||||
pub fn save(url: &str, store_path: &Path, timestamp: &str) -> Result<SaveResult> {
|
pub fn save(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<SaveResult> {
|
||||||
let single_file =
|
let single_file =
|
||||||
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
|
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
|
||||||
let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string());
|
let chrome = env::var("ARCHIVR_CHROME").unwrap_or_else(|_| "chromium".to_string());
|
||||||
save_with(url, store_path, timestamp, &single_file, &chrome)
|
save_with(url, store_path, timestamp, &single_file, &chrome, cookies)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Inner implementation; takes binary paths explicitly so tests can inject them
|
/// Inner implementation; takes binary paths explicitly so tests can inject them
|
||||||
|
|
@ -42,6 +43,7 @@ fn save_with(
|
||||||
timestamp: &str,
|
timestamp: &str,
|
||||||
single_file: &str,
|
single_file: &str,
|
||||||
chrome: &str,
|
chrome: &str,
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
) -> Result<SaveResult> {
|
) -> Result<SaveResult> {
|
||||||
let temp_dir = store_path.join("temp").join(timestamp);
|
let temp_dir = store_path.join("temp").join(timestamp);
|
||||||
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
|
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
|
||||||
|
|
@ -92,8 +94,20 @@ fn save_with(
|
||||||
.collect();
|
.collect();
|
||||||
let browser_args = format!("[{}]", quoted.join(","));
|
let browser_args = format!("[{}]", quoted.join(","));
|
||||||
|
|
||||||
let out = Command::new(single_file)
|
// Write cookie file if cookies are provided.
|
||||||
.arg(url)
|
// Never pass cookie values in process args (ps exposure).
|
||||||
|
let cookie_file: Option<std::path::PathBuf> = if !cookies.is_empty() {
|
||||||
|
let cf = temp_dir.join("cookies.txt");
|
||||||
|
let domain = domain_from_url(url);
|
||||||
|
write_netscape_cookie_file(cookies, &domain, &cf)
|
||||||
|
.context("failed to write single-file cookie file")?;
|
||||||
|
Some(cf)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut cmd = Command::new(single_file);
|
||||||
|
cmd.arg(url)
|
||||||
.arg(&out_file)
|
.arg(&out_file)
|
||||||
.arg(format!("--browser-executable-path={chrome}"))
|
.arg(format!("--browser-executable-path={chrome}"))
|
||||||
.arg("--browser-headless")
|
.arg("--browser-headless")
|
||||||
|
|
@ -120,9 +134,21 @@ fn save_with(
|
||||||
// Preserve fonts: defaults strip @font-face rules deemed "unused" or
|
// Preserve fonts: defaults strip @font-face rules deemed "unused" or
|
||||||
// "alternative" (unicode-range subsets), losing CDN-served fonts.
|
// "alternative" (unicode-range subsets), losing CDN-served fonts.
|
||||||
.arg("--remove-unused-fonts=false")
|
.arg("--remove-unused-fonts=false")
|
||||||
.arg("--remove-alternative-fonts=false")
|
.arg("--remove-alternative-fonts=false");
|
||||||
|
if let Some(cf) = &cookie_file {
|
||||||
|
cmd.arg(format!("--browser-cookies-file={}", cf.display()));
|
||||||
|
}
|
||||||
|
let spawn_result = cmd
|
||||||
.output()
|
.output()
|
||||||
.with_context(|| format!("failed to spawn {single_file} process"))?;
|
.with_context(|| format!("failed to spawn {single_file} process"));
|
||||||
|
|
||||||
|
// Delete cookie file unconditionally — including on spawn failure —
|
||||||
|
// so secrets are never left in store/temp when the capture fails.
|
||||||
|
if let Some(cf) = &cookie_file {
|
||||||
|
let _ = std::fs::remove_file(cf);
|
||||||
|
}
|
||||||
|
|
||||||
|
let out = spawn_result?;
|
||||||
|
|
||||||
if !out.status.success() {
|
if !out.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
|
|
@ -296,6 +322,7 @@ mod tests {
|
||||||
"test-ts",
|
"test-ts",
|
||||||
"/nonexistent/single-file",
|
"/nonexistent/single-file",
|
||||||
"chromium",
|
"chromium",
|
||||||
|
&HashMap::new(),
|
||||||
);
|
);
|
||||||
let err = result.unwrap_err();
|
let err = result.unwrap_err();
|
||||||
let msg = format!("{err:#}");
|
let msg = format!("{err:#}");
|
||||||
|
|
|
||||||
|
|
@ -68,7 +68,13 @@ fn build_scraper_args(
|
||||||
///
|
///
|
||||||
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
||||||
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
||||||
pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) -> Result<bool> {
|
pub fn archive(
|
||||||
|
path: &str,
|
||||||
|
thread: bool,
|
||||||
|
store_path: &Path,
|
||||||
|
timestamp: &str,
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
|
) -> Result<bool> {
|
||||||
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
||||||
// Output directory for Tweet JSON files.
|
// Output directory for Tweet JSON files.
|
||||||
let output_dir = store_path.join("raw_tweets");
|
let output_dir = store_path.join("raw_tweets");
|
||||||
|
|
@ -93,20 +99,59 @@ pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) ->
|
||||||
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
||||||
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
||||||
|
|
||||||
let credentials_file = if let Some(credentials_file) =
|
// Credentials: only use cookie rules as Twitter credentials when the resolved
|
||||||
env::var_os("ARCHIVR_TWITTER_CREDENTIALS_FILE")
|
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
|
||||||
{
|
// requires. A global cookie rule for an unrelated site must not suppress the
|
||||||
absolutize_path_from_cwd(PathBuf::from(credentials_file), &invocation_cwd)
|
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
|
||||||
|
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
|
||||||
|
// The temp file is written in the semicolon-delimited format the Python scraper
|
||||||
|
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
|
||||||
|
// subprocess returns so secrets are never left on disk on failure.
|
||||||
|
let temp_creds_path: Option<PathBuf>;
|
||||||
|
let credentials_file: PathBuf;
|
||||||
|
|
||||||
|
let has_twitter_cookies =
|
||||||
|
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||||
|
|
||||||
|
if has_twitter_cookies {
|
||||||
|
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
||||||
|
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
|
||||||
|
let creds_str = cookies
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| format!("{k}={v}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(";");
|
||||||
|
{
|
||||||
|
use std::io::Write;
|
||||||
|
#[cfg(unix)]
|
||||||
|
let mut f = {
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.write(true).create(true).truncate(true).mode(0o600)
|
||||||
|
.open(&cf)
|
||||||
|
.context("failed to write twitter credentials file")?
|
||||||
|
};
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
let mut f = std::fs::File::create(&cf)
|
||||||
|
.context("failed to write twitter credentials file")?;
|
||||||
|
f.write_all(creds_str.as_bytes())
|
||||||
|
.context("failed to write twitter credentials file")?;
|
||||||
|
}
|
||||||
|
temp_creds_path = Some(cf.clone());
|
||||||
|
credentials_file = cf;
|
||||||
|
} else if let Some(env_path) = env::var_os("ARCHIVR_TWITTER_CREDENTIALS_FILE") {
|
||||||
|
credentials_file = absolutize_path_from_cwd(PathBuf::from(env_path), &invocation_cwd);
|
||||||
|
temp_creds_path = None;
|
||||||
|
if !credentials_file.is_file() {
|
||||||
|
bail!(
|
||||||
|
"Twitter credentials file not found: {}",
|
||||||
|
credentials_file.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
bail!(
|
bail!(
|
||||||
"Twitter scraping requires ARCHIVR_TWITTER_CREDENTIALS_FILE to point to a cookies file."
|
"Twitter scraping requires either cookie rules for x.com/twitter.com \
|
||||||
);
|
or ARCHIVR_TWITTER_CREDENTIALS_FILE to be set."
|
||||||
};
|
|
||||||
|
|
||||||
if !credentials_file.is_file() {
|
|
||||||
bail!(
|
|
||||||
"Twitter credentials file not found: {}",
|
|
||||||
credentials_file.display()
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -116,12 +161,21 @@ pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) ->
|
||||||
cmd.arg(arg);
|
cmd.arg(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = cmd.output().with_context(|| {
|
// Hold the Result so we can delete the credentials file before propagating
|
||||||
|
// any error — including spawn failures.
|
||||||
|
let spawn_result = cmd.output().with_context(|| {
|
||||||
format!(
|
format!(
|
||||||
"Failed to spawn tweet scraper at {}",
|
"Failed to spawn tweet scraper at {}",
|
||||||
scraper_path.display()
|
scraper_path.display()
|
||||||
)
|
)
|
||||||
})?;
|
});
|
||||||
|
|
||||||
|
// Remove temp credentials file unconditionally — secrets must not persist.
|
||||||
|
if let Some(cf) = &temp_creds_path {
|
||||||
|
let _ = fs::remove_file(cf);
|
||||||
|
}
|
||||||
|
|
||||||
|
let output = spawn_result?;
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
|
|
@ -465,7 +519,7 @@ mod tests {
|
||||||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||||
|
|
||||||
let archived = archive("tweet:123", false, &store_path, "ts").unwrap();
|
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||||
|
|
||||||
assert!(!archived);
|
assert!(!archived);
|
||||||
|
|
||||||
|
|
@ -538,7 +592,7 @@ EOF
|
||||||
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
||||||
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
||||||
|
|
||||||
let archived = archive("tweet:123", false, &store_path, "ts").unwrap();
|
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||||
let tweet_file = output_dir.join("tweet-123.json");
|
let tweet_file = output_dir.join("tweet-123.json");
|
||||||
let contents = fs::read_to_string(&tweet_file).unwrap();
|
let contents = fs::read_to_string(&tweet_file).unwrap();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,13 @@
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{bail, Context, Result};
|
||||||
use std::{env, path::{Path, PathBuf}, process::Command};
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
env,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
process::Command,
|
||||||
|
};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
|
||||||
use crate::hash::hash_file;
|
use crate::hash::hash_file;
|
||||||
|
|
||||||
/// Returns the yt-dlp `-f` format selector for `quality`.
|
/// Returns the yt-dlp `-f` format selector for `quality`.
|
||||||
|
|
@ -96,6 +103,7 @@ pub fn download(
|
||||||
store_path: &Path,
|
store_path: &Path,
|
||||||
timestamp: &String,
|
timestamp: &String,
|
||||||
quality: Option<&str>,
|
quality: Option<&str>,
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
) -> Result<(String, String)> {
|
) -> Result<(String, String)> {
|
||||||
println!("Downloading with yt-dlp: {path}");
|
println!("Downloading with yt-dlp: {path}");
|
||||||
|
|
||||||
|
|
@ -105,6 +113,18 @@ pub fn download(
|
||||||
let temp_dir = store_path.join("temp").join(timestamp);
|
let temp_dir = store_path.join("temp").join(timestamp);
|
||||||
std::fs::create_dir_all(&temp_dir)?;
|
std::fs::create_dir_all(&temp_dir)?;
|
||||||
|
|
||||||
|
// Write a restrictive-permissions cookie file if cookies are provided.
|
||||||
|
// Never pass cookie values in process args (ps exposure).
|
||||||
|
let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
|
||||||
|
let cf_path = temp_dir.join("cookies.txt");
|
||||||
|
let domain = domain_from_url(&path);
|
||||||
|
write_netscape_cookie_file(cookies, &domain, &cf_path)
|
||||||
|
.context("failed to write yt-dlp cookie file")?;
|
||||||
|
Some(cf_path)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
// %(ext)s lets yt-dlp write the correct extension for the chosen format.
|
// %(ext)s lets yt-dlp write the correct extension for the chosen format.
|
||||||
let out_template = temp_dir.join(format!("{timestamp}.%(ext)s"));
|
let out_template = temp_dir.join(format!("{timestamp}.%(ext)s"));
|
||||||
|
|
||||||
|
|
@ -123,12 +143,21 @@ pub fn download(
|
||||||
// Force the video container to mp4 so we always have a known extension.
|
// Force the video container to mp4 so we always have a known extension.
|
||||||
cmd.arg("--merge-output-format").arg("mp4");
|
cmd.arg("--merge-output-format").arg("mp4");
|
||||||
}
|
}
|
||||||
|
if let Some(cf) = &cookie_file {
|
||||||
|
cmd.arg("--cookies").arg(cf);
|
||||||
|
}
|
||||||
let out = cmd
|
let out = cmd
|
||||||
.arg("-o")
|
.arg("-o")
|
||||||
.arg(&out_template)
|
.arg(&out_template)
|
||||||
.output()
|
.output()
|
||||||
.with_context(|| format!("failed to spawn {ytdlp} process"))?;
|
.with_context(|| format!("failed to spawn {ytdlp} process"));
|
||||||
|
|
||||||
|
// Remove cookie file immediately regardless of outcome.
|
||||||
|
if let Some(cf) = &cookie_file {
|
||||||
|
let _ = std::fs::remove_file(cf);
|
||||||
|
}
|
||||||
|
|
||||||
|
let out = out?;
|
||||||
if !out.status.success() {
|
if !out.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
bail!("yt-dlp failed: {stderr}");
|
bail!("yt-dlp failed: {stderr}");
|
||||||
|
|
@ -166,19 +195,39 @@ fn find_downloaded_file(temp_dir: &Path, timestamp: &str) -> Result<PathBuf> {
|
||||||
/// This is a simulate call — it does NOT download any media.
|
/// This is a simulate call — it does NOT download any media.
|
||||||
/// On failure (non-zero exit or no stdout), prints the captured stderr
|
/// On failure (non-zero exit or no stdout), prints the captured stderr
|
||||||
/// to stderr (for debugging) then returns `None` so callers can proceed.
|
/// to stderr (for debugging) then returns `None` so callers can proceed.
|
||||||
pub fn fetch_metadata(path: &str) -> Option<String> {
|
pub fn fetch_metadata(path: &str, cookies: &HashMap<String, String>) -> Option<String> {
|
||||||
let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
|
let ytdlp = std::env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
|
||||||
|
|
||||||
let out = std::process::Command::new(&ytdlp)
|
// Write a temp cookie file if needed; UUID-named to avoid collisions.
|
||||||
.arg("--dump-json")
|
let cookie_file: Option<PathBuf> = if !cookies.is_empty() {
|
||||||
|
let domain = domain_from_url(path);
|
||||||
|
let p = std::env::temp_dir()
|
||||||
|
.join(format!("archivr-cookies-{}.txt", Uuid::new_v4().simple()));
|
||||||
|
write_netscape_cookie_file(cookies, &domain, &p).ok()?;
|
||||||
|
Some(p)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut cmd = std::process::Command::new(&ytdlp);
|
||||||
|
cmd.arg("--dump-json")
|
||||||
// Same rationale as download(): only called for single-item sources;
|
// Same rationale as download(): only called for single-item sources;
|
||||||
// prevents --dump-json from emitting one JSON object per playlist item
|
// prevents --dump-json from emitting one JSON object per playlist item
|
||||||
// when the URL contains a list= parameter.
|
// when the URL contains a list= parameter.
|
||||||
.arg("--no-playlist")
|
.arg("--no-playlist");
|
||||||
.arg(path)
|
if let Some(cf) = &cookie_file {
|
||||||
.output()
|
cmd.arg("--cookies").arg(cf);
|
||||||
.ok()?;
|
}
|
||||||
|
cmd.arg(path);
|
||||||
|
|
||||||
|
let out = cmd.output().ok();
|
||||||
|
|
||||||
|
// Remove cookie file regardless of outcome.
|
||||||
|
if let Some(cf) = &cookie_file {
|
||||||
|
let _ = std::fs::remove_file(cf);
|
||||||
|
}
|
||||||
|
|
||||||
|
let out = out?;
|
||||||
if !out.status.success() {
|
if !out.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
eprintln!(
|
eprintln!(
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ base64.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
rusqlite.workspace = true
|
rusqlite.workspace = true
|
||||||
parking_lot.workspace = true
|
parking_lot.workspace = true
|
||||||
|
regex.workspace = true
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile.workspace = true
|
tempfile.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -264,6 +264,14 @@ pub fn app_with_state(state: AppState) -> Router {
|
||||||
"/api/admin/instance-settings",
|
"/api/admin/instance-settings",
|
||||||
get(get_instance_settings_handler).patch(update_instance_settings_handler),
|
get(get_instance_settings_handler).patch(update_instance_settings_handler),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/api/admin/cookie-rules",
|
||||||
|
get(list_cookie_rules_handler).post(create_cookie_rule_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/admin/cookie-rules/:rule_uid",
|
||||||
|
patch(update_cookie_rule_handler).delete(delete_cookie_rule_handler),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/archives/:archive_id/collections",
|
"/api/archives/:archive_id/collections",
|
||||||
get(list_collections_handler).post(create_collection_handler),
|
get(list_collections_handler).post(create_collection_handler),
|
||||||
|
|
@ -679,6 +687,21 @@ struct PatchTagBody {
|
||||||
name: String,
|
name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
struct CreateCookieRuleBody {
|
||||||
|
url_pattern: Option<String>,
|
||||||
|
pattern_kind: String,
|
||||||
|
cookies_json: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
struct UpdateCookieRuleBody {
|
||||||
|
url_pattern: Option<serde_json::Value>, // null → clear, string → set, absent → keep
|
||||||
|
pattern_kind: Option<String>,
|
||||||
|
cookies_json: Option<String>,
|
||||||
|
ordinal: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
async fn capture_handler(
|
async fn capture_handler(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
auth_user: AuthUser,
|
auth_user: AuthUser,
|
||||||
|
|
@ -710,6 +733,15 @@ async fn capture_handler(
|
||||||
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
||||||
drop(conn);
|
drop(conn);
|
||||||
|
|
||||||
|
// Load cookie rules from the auth DB to pass into the capture background task.
|
||||||
|
let cookie_rules = {
|
||||||
|
match database::open_auth_db(&state.auth_db_path) {
|
||||||
|
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
||||||
|
Err(_) => vec![],
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let capture_config = capture::CaptureConfig { cookie_rules };
|
||||||
|
|
||||||
// Spawn background capture.
|
// Spawn background capture.
|
||||||
let locator = body.locator.trim().to_string();
|
let locator = body.locator.trim().to_string();
|
||||||
let quality = body.quality.clone();
|
let quality = body.quality.clone();
|
||||||
|
|
@ -725,7 +757,7 @@ async fn capture_handler(
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok();
|
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None).ok();
|
||||||
match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref()) {
|
match capture::perform_capture(&archive_paths, &locator, Some(&archive_id_bg), quality.as_deref(), &capture_config) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
database::update_capture_job_status(
|
database::update_capture_job_status(
|
||||||
&conn,
|
&conn,
|
||||||
|
|
@ -806,8 +838,13 @@ async fn probe_handler(
|
||||||
// extractor, etc.). That is distinct from "yt-dlp ran fine but found no video": we
|
// extractor, etc.). That is distinct from "yt-dlp ran fine but found no video": we
|
||||||
// return 502 so the frontend treats it as inconclusive rather than showing
|
// return 502 so the frontend treats it as inconclusive rather than showing
|
||||||
// "No video detected" for a URL that may well be downloadable.
|
// "No video detected" for a URL that may well be downloadable.
|
||||||
|
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
|
||||||
|
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
||||||
|
Err(_) => vec![],
|
||||||
|
};
|
||||||
|
let cookies = capture::resolve_cookies_for_url(&cookie_rules, &ytdlp_url);
|
||||||
let maybe_result = tokio::task::spawn_blocking(move || {
|
let maybe_result = tokio::task::spawn_blocking(move || {
|
||||||
downloader::ytdlp::fetch_metadata(&ytdlp_url)
|
downloader::ytdlp::fetch_metadata(&ytdlp_url, &cookies)
|
||||||
.map(|json| downloader::ytdlp::probe_result(&json))
|
.map(|json| downloader::ytdlp::probe_result(&json))
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
|
@ -998,6 +1035,118 @@ async fn update_instance_settings_handler(
|
||||||
Ok(StatusCode::NO_CONTENT)
|
Ok(StatusCode::NO_CONTENT)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async fn list_cookie_rules_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth_user: AuthUser,
|
||||||
|
) -> Result<Json<Vec<database::CookieRule>>, ApiError> {
|
||||||
|
auth_user.require_role(ROLE_ADMIN)?;
|
||||||
|
let conn = database::open_auth_db(&state.auth_db_path)?;
|
||||||
|
Ok(Json(database::list_cookie_rules(&conn)?))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_cookie_rule_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth_user: AuthUser,
|
||||||
|
Json(body): Json<CreateCookieRuleBody>,
|
||||||
|
) -> Result<(StatusCode, Json<database::CookieRule>), ApiError> {
|
||||||
|
auth_user.require_role(ROLE_ADMIN)?;
|
||||||
|
if !["global", "wildcard", "regex"].contains(&body.pattern_kind.as_str()) {
|
||||||
|
return Err(ApiError::bad_request(
|
||||||
|
"pattern_kind must be 'global', 'wildcard', or 'regex'",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if serde_json::from_str::<std::collections::HashMap<String, String>>(&body.cookies_json).is_err() {
|
||||||
|
return Err(ApiError::bad_request(
|
||||||
|
"cookies_json must be a JSON object whose values are all strings, e.g. {\"name\": \"value\"}",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if body.pattern_kind != "global"
|
||||||
|
&& body.url_pattern.as_deref().unwrap_or("").trim().is_empty()
|
||||||
|
{
|
||||||
|
return Err(ApiError::bad_request(
|
||||||
|
"url_pattern is required for non-global rules",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if body.pattern_kind == "regex" {
|
||||||
|
if let Some(pat) = &body.url_pattern {
|
||||||
|
regex::Regex::new(pat)
|
||||||
|
.map_err(|e| ApiError::bad_request(&format!("invalid regex: {e}")))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let conn = database::open_auth_db(&state.auth_db_path)?;
|
||||||
|
let url_pattern = body
|
||||||
|
.url_pattern
|
||||||
|
.as_deref()
|
||||||
|
.filter(|s| !s.trim().is_empty());
|
||||||
|
let rule = database::create_cookie_rule(&conn, url_pattern, &body.pattern_kind, &body.cookies_json)?;
|
||||||
|
Ok((StatusCode::CREATED, Json(rule)))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_cookie_rule_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth_user: AuthUser,
|
||||||
|
Path(rule_uid): Path<String>,
|
||||||
|
Json(body): Json<UpdateCookieRuleBody>,
|
||||||
|
) -> Result<StatusCode, ApiError> {
|
||||||
|
auth_user.require_role(ROLE_ADMIN)?;
|
||||||
|
let conn = database::open_auth_db(&state.auth_db_path)?;
|
||||||
|
let rules = database::list_cookie_rules(&conn)?;
|
||||||
|
let existing = rules
|
||||||
|
.into_iter()
|
||||||
|
.find(|r| r.rule_uid == rule_uid)
|
||||||
|
.ok_or_else(|| ApiError::not_found("cookie rule not found"))?;
|
||||||
|
let pattern_kind = body.pattern_kind.unwrap_or(existing.pattern_kind);
|
||||||
|
let cookies_json = body.cookies_json.unwrap_or(existing.cookies_json);
|
||||||
|
let ordinal = body.ordinal.unwrap_or(existing.ordinal);
|
||||||
|
// url_pattern: null JSON value → clear, string → set, absent (None) → keep existing
|
||||||
|
let url_pattern: Option<String> = match body.url_pattern {
|
||||||
|
Some(serde_json::Value::Null) => None,
|
||||||
|
Some(serde_json::Value::String(s)) if s.trim().is_empty() => None,
|
||||||
|
Some(serde_json::Value::String(s)) => Some(s),
|
||||||
|
None => existing.url_pattern,
|
||||||
|
_ => existing.url_pattern,
|
||||||
|
};
|
||||||
|
if !["global", "wildcard", "regex"].contains(&pattern_kind.as_str()) {
|
||||||
|
return Err(ApiError::bad_request(
|
||||||
|
"pattern_kind must be 'global', 'wildcard', or 'regex'",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if serde_json::from_str::<std::collections::HashMap<String, String>>(&cookies_json).is_err() {
|
||||||
|
return Err(ApiError::bad_request(
|
||||||
|
"cookies_json must be a JSON object whose values are all strings, e.g. {\"name\": \"value\"}",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if pattern_kind == "regex" {
|
||||||
|
if let Some(pat) = &url_pattern {
|
||||||
|
regex::Regex::new(pat)
|
||||||
|
.map_err(|e| ApiError::bad_request(&format!("invalid regex: {e}")))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
database::update_cookie_rule(
|
||||||
|
&conn,
|
||||||
|
&rule_uid,
|
||||||
|
url_pattern.as_deref(),
|
||||||
|
&pattern_kind,
|
||||||
|
&cookies_json,
|
||||||
|
ordinal,
|
||||||
|
)?;
|
||||||
|
Ok(StatusCode::NO_CONTENT)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_cookie_rule_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth_user: AuthUser,
|
||||||
|
Path(rule_uid): Path<String>,
|
||||||
|
) -> Result<StatusCode, ApiError> {
|
||||||
|
auth_user.require_role(ROLE_ADMIN)?;
|
||||||
|
let conn = database::open_auth_db(&state.auth_db_path)?;
|
||||||
|
database::delete_cookie_rule(&conn, &rule_uid)
|
||||||
|
.map_err(|_| ApiError::not_found("cookie rule not found"))?;
|
||||||
|
Ok(StatusCode::NO_CONTENT)
|
||||||
|
}
|
||||||
|
|
||||||
// ── Blob / orphan cleanup ─────────────────────────────────────────────────────
|
// ── Blob / orphan cleanup ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
|
|
@ -2657,6 +2806,122 @@ mod tests {
|
||||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
async fn cookie_rules_require_admin() {
|
||||||
|
// Non-admin (no session) should get 401.
|
||||||
|
let (test_app, _dir) = make_test_app();
|
||||||
|
let response = test_app
|
||||||
|
.oneshot(Request::builder().uri("/api/admin/cookie-rules").body(Body::empty()).unwrap())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cookie_rules_create_list_delete() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (registry, _, auth_path) = make_test_registry(&dir);
|
||||||
|
let session = make_test_session(&auth_path);
|
||||||
|
|
||||||
|
let app = app(registry, auth_path);
|
||||||
|
|
||||||
|
// Create a global rule with valid string-only cookies.
|
||||||
|
let create_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/api/admin/cookie-rules")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.header("cookie", &session)
|
||||||
|
.body(Body::from(r#"{"pattern_kind":"global","cookies_json":"{\"session\":\"abc\"}"}"#))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(create_resp.status(), StatusCode::CREATED);
|
||||||
|
let body: serde_json::Value = serde_json::from_slice(
|
||||||
|
&axum::body::to_bytes(create_resp.into_body(), usize::MAX).await.unwrap(),
|
||||||
|
).unwrap();
|
||||||
|
let rule_uid = body["rule_uid"].as_str().unwrap().to_string();
|
||||||
|
assert_eq!(body["pattern_kind"], "global");
|
||||||
|
|
||||||
|
// List: should contain the created rule.
|
||||||
|
let list_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.uri("/api/admin/cookie-rules")
|
||||||
|
.header("cookie", &session)
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(list_resp.status(), StatusCode::OK);
|
||||||
|
let list: serde_json::Value = serde_json::from_slice(
|
||||||
|
&axum::body::to_bytes(list_resp.into_body(), usize::MAX).await.unwrap(),
|
||||||
|
).unwrap();
|
||||||
|
assert_eq!(list.as_array().unwrap().len(), 1);
|
||||||
|
|
||||||
|
// Delete.
|
||||||
|
let del_resp = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("DELETE")
|
||||||
|
.uri(format!("/api/admin/cookie-rules/{rule_uid}"))
|
||||||
|
.header("cookie", &session)
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(del_resp.status(), StatusCode::NO_CONTENT);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cookie_rules_rejects_non_string_values() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (registry, _, auth_path) = make_test_registry(&dir);
|
||||||
|
let session = make_test_session(&auth_path);
|
||||||
|
|
||||||
|
// cookies_json with a numeric value must be rejected — core only accepts string values.
|
||||||
|
let resp = app(registry, auth_path)
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/api/admin/cookie-rules")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.header("cookie", &session)
|
||||||
|
.body(Body::from(r#"{"pattern_kind":"global","cookies_json":"{\"session\":123}"}"#))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cookie_rules_rejects_invalid_regex() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (registry, _, auth_path) = make_test_registry(&dir);
|
||||||
|
let session = make_test_session(&auth_path);
|
||||||
|
|
||||||
|
let resp = app(registry, auth_path)
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/api/admin/cookie-rules")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.header("cookie", &session)
|
||||||
|
.body(Body::from(r#"{"pattern_kind":"regex","url_pattern":"[invalid","cookies_json":"{\"x\":\"y\"}"}"#))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||||
|
}
|
||||||
|
#[tokio::test]
|
||||||
async fn security_headers_present_on_success_response() {
|
async fn security_headers_present_on_success_response() {
|
||||||
let (test_app, _dir) = make_test_app();
|
let (test_app, _dir) = make_test_app();
|
||||||
let response = test_app
|
let response = test_app
|
||||||
|
|
|
||||||
40
crates/archivr-server/static/assets/index-CHDuICqH.js
Normal file
40
crates/archivr-server/static/assets/index-CHDuICqH.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4,7 +4,7 @@
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>Archivr</title>
|
<title>Archivr</title>
|
||||||
<script type="module" crossorigin src="/assets/index-D2gm2yNL.js"></script>
|
<script type="module" crossorigin src="/assets/index-CHDuICqH.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|
|
||||||
|
|
@ -365,6 +365,49 @@ export async function deleteOrphanBlobs(archiveId) {
|
||||||
return res.json()
|
return res.json()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function listCookieRules() {
|
||||||
|
return getJson('/api/admin/cookie-rules')
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createCookieRule(urlPattern, patternKind, cookiesJson) {
|
||||||
|
const res = await fetch('/api/admin/cookie-rules', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
url_pattern: urlPattern || null,
|
||||||
|
pattern_kind: patternKind,
|
||||||
|
cookies_json: cookiesJson,
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.json().catch(() => ({}))
|
||||||
|
throw new Error(body.message || `HTTP ${res.status}`)
|
||||||
|
}
|
||||||
|
return res.json()
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateCookieRule(ruleUid, patch) {
|
||||||
|
const res = await fetch(`/api/admin/cookie-rules/${ruleUid}`, {
|
||||||
|
method: 'PATCH',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(patch),
|
||||||
|
})
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.json().catch(() => ({}))
|
||||||
|
throw new Error(body.message || `HTTP ${res.status}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteCookieRule(ruleUid) {
|
||||||
|
const res = await fetch(`/api/admin/cookie-rules/${ruleUid}`, { method: 'DELETE' })
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.json().catch(() => ({}))
|
||||||
|
throw new Error(body.message || `HTTP ${res.status}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── 401 interceptor ───────────────────────────────────────────────────────────
|
// ── 401 interceptor ───────────────────────────────────────────────────────────
|
||||||
const _origFetch = window.fetch;
|
const _origFetch = window.fetch;
|
||||||
window.fetch = async (...args) => {
|
window.fetch = async (...args) => {
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import {
|
||||||
listTokens, createToken, deleteToken,
|
listTokens, createToken, deleteToken,
|
||||||
getInstanceSettings, updateInstanceSettings,
|
getInstanceSettings, updateInstanceSettings,
|
||||||
scanOrphanBlobs, deleteOrphanBlobs,
|
scanOrphanBlobs, deleteOrphanBlobs,
|
||||||
|
listCookieRules, createCookieRule, updateCookieRule, deleteCookieRule,
|
||||||
} from '../api.js'
|
} from '../api.js'
|
||||||
|
|
||||||
const ROLE_ADMIN = 4
|
const ROLE_ADMIN = 4
|
||||||
|
|
@ -13,8 +14,8 @@ export default function SettingsView({ tab, onTabChange, archiveId }) {
|
||||||
const { currentUser, setCurrentUser } = useContext(AuthContext) ?? {}
|
const { currentUser, setCurrentUser } = useContext(AuthContext) ?? {}
|
||||||
const isAdmin = currentUser && ((currentUser.role_bits & ROLE_ADMIN) !== 0)
|
const isAdmin = currentUser && ((currentUser.role_bits & ROLE_ADMIN) !== 0)
|
||||||
|
|
||||||
const tabs = ['profile', 'tokens', ...(isAdmin ? ['instance', 'storage'] : [])]
|
const tabs = ['profile', 'tokens', ...(isAdmin ? ['instance', 'cookies', 'storage'] : [])]
|
||||||
const tabLabels = { profile: 'Profile', tokens: 'API Tokens', instance: 'Instance', storage: 'Storage' }
|
const tabLabels = { profile: 'Profile', tokens: 'API Tokens', instance: 'Instance', cookies: 'Cookies', storage: 'Storage' }
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<section className="admin-view">
|
<section className="admin-view">
|
||||||
|
|
@ -32,6 +33,7 @@ export default function SettingsView({ tab, onTabChange, archiveId }) {
|
||||||
{tab === 'profile' && <ProfileTab currentUser={currentUser} setCurrentUser={setCurrentUser} />}
|
{tab === 'profile' && <ProfileTab currentUser={currentUser} setCurrentUser={setCurrentUser} />}
|
||||||
{tab === 'tokens' && <TokensTab />}
|
{tab === 'tokens' && <TokensTab />}
|
||||||
{tab === 'instance' && isAdmin && <InstanceTab />}
|
{tab === 'instance' && isAdmin && <InstanceTab />}
|
||||||
|
{tab === 'cookies' && isAdmin && <CookiesTab />}
|
||||||
{tab === 'storage' && isAdmin && <StorageTab archiveId={archiveId} />}
|
{tab === 'storage' && isAdmin && <StorageTab archiveId={archiveId} />}
|
||||||
</section>
|
</section>
|
||||||
)
|
)
|
||||||
|
|
@ -430,3 +432,210 @@ function StorageTab({ archiveId }) {
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function CookiesTab() {
|
||||||
|
const [rules, setRules] = useState(null)
|
||||||
|
const [loading, setLoading] = useState(true)
|
||||||
|
const [error, setError] = useState(null)
|
||||||
|
|
||||||
|
// Form state for adding a new rule
|
||||||
|
const [patternKind, setPatternKind] = useState('global')
|
||||||
|
const [urlPattern, setUrlPattern] = useState('')
|
||||||
|
const [cookiesInput, setCookiesInput] = useState('{}')
|
||||||
|
const [addMsg, setAddMsg] = useState(null)
|
||||||
|
const [adding, setAdding] = useState(false)
|
||||||
|
|
||||||
|
// Inline-edit state: ruleUid → { cookiesInput, saving, msg }
|
||||||
|
const [edits, setEdits] = useState({})
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
load()
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
setLoading(true); setError(null)
|
||||||
|
try { setRules(await listCookieRules()) }
|
||||||
|
catch (e) { setError(e.message) }
|
||||||
|
finally { setLoading(false) }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleAdd(e) {
|
||||||
|
e.preventDefault()
|
||||||
|
// Validate JSON
|
||||||
|
try { JSON.parse(cookiesInput) } catch {
|
||||||
|
setAddMsg({ ok: false, text: 'cookies must be valid JSON, e.g. {"session": "abc"}' })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
setAdding(true); setAddMsg(null)
|
||||||
|
try {
|
||||||
|
await createCookieRule(
|
||||||
|
patternKind === 'global' ? null : urlPattern.trim(),
|
||||||
|
patternKind,
|
||||||
|
cookiesInput,
|
||||||
|
)
|
||||||
|
setUrlPattern('')
|
||||||
|
setCookiesInput('{}')
|
||||||
|
setPatternKind('global')
|
||||||
|
setAddMsg({ ok: true, text: 'Rule added.' })
|
||||||
|
await load()
|
||||||
|
} catch (err) {
|
||||||
|
setAddMsg({ ok: false, text: err.message })
|
||||||
|
} finally {
|
||||||
|
setAdding(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleDelete(ruleUid) {
|
||||||
|
try {
|
||||||
|
await deleteCookieRule(ruleUid)
|
||||||
|
await load()
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function startEdit(rule) {
|
||||||
|
setEdits(prev => ({
|
||||||
|
...prev,
|
||||||
|
[rule.rule_uid]: {
|
||||||
|
cookiesInput: rule.cookies_json,
|
||||||
|
saving: false,
|
||||||
|
msg: null,
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
function cancelEdit(ruleUid) {
|
||||||
|
setEdits(prev => { const n = { ...prev }; delete n[ruleUid]; return n })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveEdit(rule) {
|
||||||
|
const edit = edits[rule.rule_uid]
|
||||||
|
try { JSON.parse(edit.cookiesInput) } catch {
|
||||||
|
setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], msg: { ok: false, text: 'Invalid JSON' } } }))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], saving: true, msg: null } }))
|
||||||
|
try {
|
||||||
|
await updateCookieRule(rule.rule_uid, { cookies_json: edit.cookiesInput })
|
||||||
|
cancelEdit(rule.rule_uid)
|
||||||
|
await load()
|
||||||
|
} catch (err) {
|
||||||
|
setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], saving: false, msg: { ok: false, text: err.message } } }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (loading) return <div className="muted">Loading…</div>
|
||||||
|
if (error) return <div className="form-msg form-msg--err">{error}</div>
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ maxWidth: 560 }}>
|
||||||
|
<div className="form-section">
|
||||||
|
<h2>Cookie Rules</h2>
|
||||||
|
<p className="muted" style={{ marginBottom: 12 }}>
|
||||||
|
Cookies are injected into every capture network request (yt-dlp, HTTP downloads, web-page snapshots).
|
||||||
|
Global rules apply to all URLs; wildcard and regex rules apply only to matching URLs.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{rules && rules.length > 0 ? (
|
||||||
|
<table className="data-table" style={{ width: '100%', marginBottom: 16 }}>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Pattern</th>
|
||||||
|
<th>Cookies</th>
|
||||||
|
<th style={{ width: 100 }}>Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{rules.map(rule => {
|
||||||
|
const edit = edits[rule.rule_uid]
|
||||||
|
const patternLabel = rule.url_pattern
|
||||||
|
? <><span className="muted">{rule.pattern_kind}:</span> <code>{rule.url_pattern}</code></>
|
||||||
|
: <span className="muted">global (all URLs)</span>
|
||||||
|
return (
|
||||||
|
<tr key={rule.rule_uid}>
|
||||||
|
<td>{patternLabel}</td>
|
||||||
|
<td>
|
||||||
|
{edit ? (
|
||||||
|
<>
|
||||||
|
<textarea
|
||||||
|
className="field-input"
|
||||||
|
style={{ fontFamily: 'monospace', fontSize: 12, width: '100%', minHeight: 60 }}
|
||||||
|
value={edit.cookiesInput}
|
||||||
|
onChange={ev => setEdits(prev => ({ ...prev, [rule.rule_uid]: { ...prev[rule.rule_uid], cookiesInput: ev.target.value } }))}
|
||||||
|
/>
|
||||||
|
{edit.msg && <div className={`form-msg form-msg--${edit.msg.ok ? 'ok' : 'err'}`}>{edit.msg.text}</div>}
|
||||||
|
<div style={{ display: 'flex', gap: 8, marginTop: 4 }}>
|
||||||
|
<button className="btn-primary" style={{ fontSize: 12, padding: '2px 8px' }} disabled={edit.saving} onClick={() => saveEdit(rule)}>
|
||||||
|
{edit.saving ? 'Saving\u2026' : 'Save'}
|
||||||
|
</button>
|
||||||
|
<button className="btn-secondary" style={{ fontSize: 12, padding: '2px 8px' }} onClick={() => cancelEdit(rule.rule_uid)}>Cancel</button>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<code style={{ fontSize: 12, wordBreak: 'break-all' }}>{rule.cookies_json}</code>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{!edit && (
|
||||||
|
<div style={{ display: 'flex', gap: 6 }}>
|
||||||
|
<button className="btn-secondary" style={{ fontSize: 12, padding: '2px 8px' }} onClick={() => startEdit(rule)}>Edit</button>
|
||||||
|
<button className="btn-danger" style={{ fontSize: 12, padding: '2px 8px' }} onClick={() => handleDelete(rule.rule_uid)}>Del</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
) : (
|
||||||
|
<p className="muted" style={{ marginBottom: 16 }}>No cookie rules defined.</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<h3 style={{ marginBottom: 8 }}>Add Rule</h3>
|
||||||
|
<form onSubmit={handleAdd}>
|
||||||
|
<div className="form-field">
|
||||||
|
<label className="form-label">Pattern type</label>
|
||||||
|
<select className="field-input" value={patternKind} onChange={e => setPatternKind(e.target.value)}>
|
||||||
|
<option value="global">Global (all URLs)</option>
|
||||||
|
<option value="wildcard">Wildcard (e.g. *.youtube.com)</option>
|
||||||
|
<option value="regex">Regex (matched against full URL)</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
{patternKind !== 'global' && (
|
||||||
|
<div className="form-field">
|
||||||
|
<label className="form-label">
|
||||||
|
{patternKind === 'wildcard' ? 'URL/hostname pattern' : 'Regex pattern'}
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
className="field-input"
|
||||||
|
type="text"
|
||||||
|
value={urlPattern}
|
||||||
|
onChange={e => setUrlPattern(e.target.value)}
|
||||||
|
placeholder={patternKind === 'wildcard' ? '*.youtube.com or https://example.com/*' : '.*\\.youtube\\.com.*'}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="form-field">
|
||||||
|
<label className="form-label">Cookies (JSON object)</label>
|
||||||
|
<textarea
|
||||||
|
className="field-input"
|
||||||
|
style={{ fontFamily: 'monospace', fontSize: 13, minHeight: 70 }}
|
||||||
|
value={cookiesInput}
|
||||||
|
onChange={e => setCookiesInput(e.target.value)}
|
||||||
|
placeholder='{"SESSION": "abc123", "token": "xyz"}'
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{addMsg && <div className={`form-msg form-msg--${addMsg.ok ? 'ok' : 'err'}`}>{addMsg.text}</div>}
|
||||||
|
<button className="btn-primary" type="submit" disabled={adding}>
|
||||||
|
{adding ? 'Adding\u2026' : 'Add Rule'}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue