1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 03:05:32 +02:00

Compare commits

..

No commits in common. "2779afee2dd36271c20a5384c98cfc71f0220885" and "2e8820a0dac027f70bcd364877b880111d0ab8ec" have entirely different histories.

16 changed files with 215 additions and 675 deletions

View file

@ -742,52 +742,6 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
} }
} }
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
fn register_tweet_artifacts(
conn: &rusqlite::Connection,
store_path: &Path,
entry_id: i64,
tweet_json_relpaths: &[String],
) -> Result<()> {
for relpath in tweet_json_relpaths {
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id,
artifact_role: "raw_tweet_json".to_string(),
storage_area: "raw_tweets".to_string(),
relpath: relpath.clone(),
blob_id: None,
logical_path: None,
metadata_json: None,
},
)?;
let json_path = store_path.join(relpath);
let json_str = fs::read_to_string(&json_path)
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
let raw_path = PathBuf::from(&raw_relpath);
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
let blob_id = database::upsert_blob(conn, &blob)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id,
artifact_role: role,
storage_area: "raw".to_string(),
relpath: raw_relpath,
blob_id: Some(blob_id),
logical_path: None,
metadata_json: None,
},
)?;
}
}
Ok(())
}
fn record_tweet_entry( fn record_tweet_entry(
conn: &rusqlite::Connection, conn: &rusqlite::Connection,
store_path: &Path, store_path: &Path,
@ -797,7 +751,6 @@ fn record_tweet_entry(
requested_locator: &str, requested_locator: &str,
source: Source, source: Source,
tweet_id: &str, tweet_id: &str,
tweet_json_relpaths: &[String],
) -> Result<database::ArchivedEntry> { ) -> Result<database::ArchivedEntry> {
debug_assert!(run.run_uid.starts_with("run_")); debug_assert!(run.run_uid.starts_with("run_"));
debug_assert!(item.item_uid.starts_with("item_")); debug_assert!(item.item_uid.starts_with("item_"));
@ -811,7 +764,7 @@ fn record_tweet_entry(
Some(&canonical_locator), Some(&canonical_locator),
&canonical_locator, &canonical_locator,
)?; )?;
// Read the primary tweet JSON to extract title before entry creation. // Read tweet JSON early to extract title before entry creation
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
let tweet_meta = tweet_metadata_from_json(&tweet_json); let tweet_meta = tweet_metadata_from_json(&tweet_json);
@ -841,8 +794,36 @@ fn record_tweet_entry(
)?; )?;
create_structured_root(store_path, &entry)?; create_structured_root(store_path, &entry)?;
// Register all tweet JSONs and their blobs. database::add_entry_artifact(
register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?; conn,
&database::NewArtifact {
entry_id: entry.id,
artifact_role: "raw_tweet_json".to_string(),
storage_area: "raw_tweets".to_string(),
relpath: path_to_store_string(&tweet_json_relpath),
blob_id: None,
logical_path: None,
metadata_json: None,
},
)?;
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
let raw_path = PathBuf::from(&raw_relpath);
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
let blob_id = database::upsert_blob(conn, &blob)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id: entry.id,
artifact_role: role,
storage_area: "raw".to_string(),
relpath: raw_relpath,
blob_id: Some(blob_id),
logical_path: None,
metadata_json: None,
},
)?;
}
database::complete_archive_run_item(conn, item.id, entry.id)?; database::complete_archive_run_item(conn, item.id, entry.id)?;
Ok(entry) Ok(entry)
@ -919,12 +900,14 @@ pub fn perform_capture(
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage, Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
Ok(downloader::http::UrlKind::File) => {} Ok(downloader::http::UrlKind::File) => {}
Err(e) => { Err(e) => {
// Probe failed (e.g. Cloudflare JS challenge, 403, network // Record a failed item using the pre-probe source_kind so
// issue). Fall back to treating the URL as an HTML page so // failed_count increments and the item carries error_text.
// that the SingleFile/Chromium path can try — a real browser let (probe_sk, probe_ek, _) = source_metadata(Source::Url);
// can pass bot challenges that a plain HTTP client cannot. let item = database::create_archive_run_item(
eprintln!("warn: probe failed for {locator}, assuming HTML: {e}"); &conn, run.id, None, 0, locator, None, probe_sk, probe_ek,
source = Source::WebPage; )?;
let msg = format!("Failed to probe URL: {e}");
return Err(fail_run(&conn, &run, &item, &msg));
} }
} }
} }
@ -1203,7 +1186,7 @@ pub fn perform_capture(
&timestamp, &timestamp,
&tweet_cookies, &tweet_cookies,
) { ) {
Ok(tweet_json_relpaths) => { Ok(_) => {
let tweet_entry = record_tweet_entry( let tweet_entry = record_tweet_entry(
&conn, &conn,
store_path, store_path,
@ -1213,7 +1196,6 @@ pub fn perform_capture(
locator, locator,
source, source,
&tweet_id, &tweet_id,
&tweet_json_relpaths,
)?; )?;
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?; database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
@ -1382,107 +1364,6 @@ pub fn perform_capture(
}) })
} }
/// Result of a tweet re-archive operation.
#[derive(Debug, serde::Serialize)]
pub struct RearchiveResult {
/// One of: "completed", "not_a_tweet", "scraper_failed".
pub status: String,
/// Human-readable detail (empty string for "completed").
pub message: String,
}
/// Re-archives an existing tweet or tweet_thread entry in-place.
///
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
/// returns `scraper_failed` WITHOUT touching the existing data.
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
/// and their media blobs. Does not change archived_at, title, tags, or collections.
pub fn perform_rearchive(
archive_paths: &ArchivePaths,
entry_uid: &str,
config: &CaptureConfig,
) -> Result<RearchiveResult> {
let store_path = &archive_paths.store_path;
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
// Look up the entry.
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
None => {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: "entry not found".to_string(),
});
}
Some(e) => e,
};
// Only tweet / tweet_thread entries can be re-archived this way.
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: format!("entry is '{}', not a tweet", entry.entity_kind),
});
}
// Parse source_metadata_json for tweet_id and requested_locator.
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
.unwrap_or(serde_json::Value::Object(Default::default()));
let requested_locator = meta["requested_locator"]
.as_str()
.unwrap_or("")
.to_string();
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
if requested_locator.is_empty() || tweet_id.is_empty() {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
});
}
let is_thread = entry.entity_kind == "tweet_thread";
let timestamp = format!(
"{}-{}",
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
uuid::Uuid::new_v4().simple(),
);
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
// Run the scraper (staged). If this fails, existing data is untouched.
let tweet_json_relpaths = match downloader::tweets::rearchive(
&requested_locator,
is_thread,
store_path,
&timestamp,
&tweet_cookies,
) {
Ok(relpaths) => relpaths,
Err(e) => {
return Ok(RearchiveResult {
status: "scraper_failed".to_string(),
message: format!("{e:#}"),
});
}
};
// Atomically swap artifact rows: delete old, insert new.
{
let tx = conn.transaction()?;
database::delete_entry_artifacts(&tx, entry.id)?;
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
tx.commit()?;
}
database::refresh_entry_cached_bytes(&conn, entry.id)?;
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
Ok(RearchiveResult {
status: "completed".to_string(),
message: String::new(),
})
}
fn mime_for_font_ext(ext: &str) -> Option<String> { fn mime_for_font_ext(ext: &str) -> Option<String> {
match ext { match ext {
".woff2" => Some("font/woff2".to_string()), ".woff2" => Some("font/woff2".to_string()),
@ -2109,7 +1990,6 @@ mod tests {
"tweet:123", "tweet:123",
Source::Tweet, Source::Tweet,
"123", "123",
&["raw_tweets/tweet-123.json".to_string()],
) )
.unwrap(); .unwrap();
database::finish_archive_run(&conn, run.id).unwrap(); database::finish_archive_run(&conn, run.id).unwrap();

View file

@ -2391,43 +2391,6 @@ fn humanize_slug(slug: &str) -> String {
.join(" ") .join(" ")
} }
/// A minimal view of an archived entry needed for re-archiving.
pub struct EntryForRearchive {
pub id: i64,
pub entity_kind: String,
pub source_metadata_json: String,
}
/// Looks up an entry by its public UID for re-archive purposes.
pub fn get_entry_for_rearchive(
conn: &Connection,
entry_uid: &str,
) -> Result<Option<EntryForRearchive>> {
conn.query_row(
"SELECT id, entity_kind, source_metadata_json \
FROM archived_entries WHERE entry_uid = ?1",
[entry_uid],
|row| {
Ok(EntryForRearchive {
id: row.get(0)?,
entity_kind: row.get(1)?,
source_metadata_json: row.get(2)?,
})
},
)
.optional()
.map_err(anyhow::Error::from)
}
/// Deletes all `entry_artifacts` rows for the given entry.
/// Call within a transaction for atomicity with re-insertion.
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
Ok(conn.execute(
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
[entry_id],
)?)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

View file

@ -20,7 +20,7 @@ pub enum UrlKind {
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> { pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10)) .redirect(reqwest::redirect::Policy::limited(10))
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1") .user_agent("archivr/0.1")
.build() .build()
.context("failed to build HTTP client")?; .context("failed to build HTTP client")?;
@ -86,7 +86,7 @@ pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<Ur
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> { pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder()
.redirect(reqwest::redirect::Policy::limited(10)) .redirect(reqwest::redirect::Policy::limited(10))
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1") .user_agent("archivr/0.1")
.build() .build()
.context("failed to build HTTP client")?; .context("failed to build HTTP client")?;

View file

@ -1,7 +1,7 @@
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use regex::Regex; use regex::Regex;
use std::{ use std::{
collections::HashMap, collections::{HashMap, HashSet},
env, env,
ffi::OsString, ffi::OsString,
fs, fs,
@ -59,39 +59,63 @@ fn build_scraper_args(
args args
} }
/// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs /// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
/// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`). ///
/// The staging dir starts empty so all files found there are exactly the touched set. /// Invokes the Python scraper, then moves all produced media assets into the
fn run_scraper_staged( /// content-addressed raw store and rewrites the JSON output to use the new
tweet_id: &str, /// store-relative paths. Returns `true` if new content was archived, `false`
/// if the tweet was already present and `thread` is `false`.
///
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
pub fn archive(
path: &str,
thread: bool, thread: bool,
store_path: &Path, store_path: &Path,
timestamp: &str, timestamp: &str,
cookies: &HashMap<String, String>, cookies: &HashMap<String, String>,
) -> Result<Vec<String>> { ) -> Result<bool> {
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?; let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
// Output directory for Tweet JSON files.
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
// Scraper media-dir = staging_dir/media/
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
let staging_tweets_dir = staging_dir.join("raw_tweets");
fs::create_dir_all(&staging_tweets_dir)?;
// Final destination for tweet JSONs.
let output_dir = store_path.join("raw_tweets"); let output_dir = store_path.join("raw_tweets");
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
fs::create_dir_all(&output_dir)?; fs::create_dir_all(&output_dir)?;
fs::create_dir_all(&temp_dir)?;
// Path to the root - the to-be-archived tweet's JSON file.
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
if !thread && root_json.exists() {
return Ok(false);
}
let before = tweet_json_files(&output_dir)?;
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3")); let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER") let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
.map(PathBuf::from) .map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py")); .unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd); let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
// Credentials: only use cookie rules as Twitter credentials when the resolved
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
// requires. A global cookie rule for an unrelated site must not suppress the
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
// The temp file is written in the semicolon-delimited format the Python scraper
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
// subprocess returns so secrets are never left on disk on failure.
let temp_creds_path: Option<PathBuf>; let temp_creds_path: Option<PathBuf>;
let credentials_file: PathBuf; let credentials_file: PathBuf;
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
let has_twitter_cookies =
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
if has_twitter_cookies { if has_twitter_cookies {
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt"); let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
let creds_str = cookies let creds_str = cookies
.iter() .iter()
.map(|(k, v)| format!("{k}={v}")) .map(|(k, v)| format!("{k}={v}"))
@ -111,7 +135,7 @@ fn run_scraper_staged(
let mut f = std::fs::File::create(&cf) let mut f = std::fs::File::create(&cf)
.context("failed to write twitter credentials file")?; .context("failed to write twitter credentials file")?;
f.write_all(creds_str.as_bytes()) f.write_all(creds_str.as_bytes())
.context("failed to write twitter credentials file")? .context("failed to write twitter credentials file")?;
} }
temp_creds_path = Some(cf.clone()); temp_creds_path = Some(cf.clone());
credentials_file = cf; credentials_file = cf;
@ -132,21 +156,28 @@ fn run_scraper_staged(
} }
let mut cmd = Command::new(&python); let mut cmd = Command::new(&python);
// Run the scraper from staging_dir so relative paths in JSON resolve correctly. cmd.current_dir(&temp_dir).arg(&scraper_path);
cmd.current_dir(&staging_dir).arg(&scraper_path); for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
cmd.arg(arg); cmd.arg(arg);
} }
// Hold the Result so we can delete the credentials file before propagating
// any error — including spawn failures.
let spawn_result = cmd.output().with_context(|| { let spawn_result = cmd.output().with_context(|| {
format!("Failed to spawn tweet scraper at {}", scraper_path.display()) format!(
"Failed to spawn tweet scraper at {}",
scraper_path.display()
)
}); });
// Remove temp credentials file unconditionally — secrets must not persist.
if let Some(cf) = &temp_creds_path { if let Some(cf) = &temp_creds_path {
let _ = fs::remove_file(cf); let _ = fs::remove_file(cf);
} }
let output = spawn_result?; let output = spawn_result?;
if !output.status.success() { if !output.status.success() {
let _ = fs::remove_dir_all(&staging_dir);
let stderr = String::from_utf8_lossy(&output.stderr); let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
bail!( bail!(
@ -156,9 +187,7 @@ fn run_scraper_staged(
); );
} }
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
if !root_json.exists() { if !root_json.exists() {
let _ = fs::remove_dir_all(&staging_dir);
let stderr = String::from_utf8_lossy(&output.stderr); let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
bail!( bail!(
@ -169,82 +198,13 @@ fn run_scraper_staged(
); );
} }
// Remove the scraping_summary.json if the scraper left one. cleanup_summary(&output_dir)?;
cleanup_summary(&staging_tweets_dir)?; let after = tweet_json_files(&output_dir)?;
let new_jsons = new_tweet_jsons(&before, &after);
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
// Collect all tweet-*.json files from staging (this is the exact touched set). Ok(true)
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
})
.collect();
staged_jsons.sort();
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
// Move each staged JSON to its final destination, collecting store-relative relpaths.
let mut relpaths = Vec::with_capacity(staged_jsons.len());
for staged_path in &staged_jsons {
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
let dest = output_dir.join(filename);
fs::rename(staged_path, &dest)
.or_else(|_| {
fs::copy(staged_path, &dest).map(|_| ())?;
fs::remove_file(staged_path)
})
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
let rel = dest
.strip_prefix(store_path)
.context("dest tweet JSON is not under store_path")?;
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
}
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
let _ = fs::remove_dir_all(&staging_dir);
Ok(relpaths)
}
/// Archives a tweet (or full thread) identified by `path`.
///
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
/// For single tweets already on disk, returns the existing file path without re-downloading.
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
pub fn archive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<Vec<String>> {
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
if !thread && root_json.exists() {
// Already present; skip re-download, return the existing file for artifact registration.
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
}
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
}
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
/// Always runs the scraper; output is staged before replacing existing files.
/// Returns store-relative relpaths of all produced tweet JSON files.
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
pub fn rearchive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<Vec<String>> {
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
} }
/// Removes the `scraping_summary.json` file left by the scraper, if present. /// Removes the `scraping_summary.json` file left by the scraper, if present.
@ -256,6 +216,33 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
Ok(()) Ok(())
} }
/// Returns the set of `tweet-*.json` files present in `output_dir`.
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
let mut files = HashSet::new();
for entry in fs::read_dir(output_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_file()
&& path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
{
files.insert(path);
}
}
Ok(files)
}
/// Returns the sorted list of JSON files present in `after` but not in `before`.
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
files.sort();
files
}
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON. /// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
fn avatar_regex() -> &'static Regex { fn avatar_regex() -> &'static Regex {
@ -532,9 +519,9 @@ mod tests {
fs::write(&credentials, "ct0=test;auth_token=test").unwrap(); fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials); set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap(); let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]); assert!(!archived);
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE"); remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
let _ = fs::remove_dir_all(store_path); let _ = fs::remove_dir_all(store_path);
@ -589,7 +576,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
{ {
"id": "$tweet_id", "id": "$tweet_id",
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] }, "entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
"author": { "avatar_local_path": "$media_dir/avatars/author.jpg" } "author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
} }
EOF EOF
"#, "#,
@ -605,16 +592,16 @@ EOF
set_test_env("ARCHIVR_TWEET_SCRAPER", &script); set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh"); set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap(); let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
let tweet_file = output_dir.join("tweet-123.json"); let tweet_file = output_dir.join("tweet-123.json");
let contents = fs::read_to_string(&tweet_file).unwrap(); let contents = fs::read_to_string(&tweet_file).unwrap();
assert!(!relpaths.is_empty()); assert!(archived);
assert!(tweet_file.exists()); assert!(tweet_file.exists());
assert!(!output_dir.join("scraping_summary.json").exists()); assert!(!output_dir.join("scraping_summary.json").exists());
assert!(contents.contains(r#""avatar_local_path": "raw/"#)); assert!(contents.contains(r#""avatar_local_path": "raw/"#));
assert!(contents.contains(r#""local_path": "raw/"#)); assert!(contents.contains(r#""local_path": "raw/"#));
assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists()); assert!(!store_path.join("temp").join("ts").exists());
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE"); remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
remove_test_env("ARCHIVR_TWEET_SCRAPER"); remove_test_env("ARCHIVR_TWEET_SCRAPER");

View file

@ -221,10 +221,6 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index", "/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
get(serve_artifact), get(serve_artifact),
) )
.route(
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
post(rearchive_handler),
)
.route( .route(
"/api/archives/:archive_id/entries/:entry_uid/favicon", "/api/archives/:archive_id/entries/:entry_uid/favicon",
get(serve_entry_favicon), get(serve_entry_favicon),
@ -836,83 +832,6 @@ async fn get_capture_job_handler(
.ok_or_else(|| ApiError::not_found("capture job not found")) .ok_or_else(|| ApiError::not_found("capture job not found"))
} }
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
///
/// Re-archives an existing tweet or tweet_thread entry in-place:
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
///
/// Returns 202 immediately with a job_uid the client should poll via
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
async fn rearchive_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path((archive_id, entry_uid)): Path<(String, String)>,
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
auth_user.require_role(ROLE_USER)?;
let mounted = mounted_archive(&state, &archive_id)?;
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
.map_err(ApiError::from)?;
// Create a capture job record so the client can poll for completion.
let conn = database::open_or_initialize(&mounted.archive_path)?;
let job_uid = database::create_capture_job(&conn, &archive_id)?;
drop(conn);
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
Err(_) => vec![],
};
let capture_config = capture::CaptureConfig {
cookie_rules,
ublock_enabled: None,
cookie_ext_enabled: None,
reader_mode: false,
};
let job_uid_bg = job_uid.clone();
let archive_path = mounted.archive_path.clone();
tokio::task::spawn_blocking(move || {
let conn = match database::open_or_initialize(&archive_path) {
Ok(c) => c,
Err(e) => {
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
return;
}
};
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
Ok(result) => {
if result.status == "completed" {
database::update_capture_job_status(
&conn, &job_uid_bg, "completed", None, None, None,
).ok();
} else {
// "not_a_tweet" or "scraper_failed" — surface as a job failure
// so the client sees a meaningful error message.
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&result.message), None,
).ok();
}
}
Err(e) => {
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&format!("{e:#}")), None,
).ok();
}
}
});
Ok((
StatusCode::ACCEPTED,
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
))
}
/// `GET /api/archives/:id/captures/probe?locator=<url>` /// `GET /api/archives/:id/captures/probe?locator=<url>`
/// ///
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video /// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,8 +4,8 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title> <title>Archivr</title>
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script> <script type="module" crossorigin src="/assets/index-D6iX-c2T.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css"> <link rel="stylesheet" crossorigin href="/assets/index-C0BIEdow.css">
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

View file

@ -241,12 +241,10 @@ export default function App() {
]) ])
}, [archiveId, searchQuery, tagFilter, loadEntries]) }, [archiveId, searchQuery, tagFilter, loadEntries])
const handleToast = useCallback((text, locator, type = 'error', headline = null) => { const handleToast = useCallback((text, locator, type = 'error') => {
// Only suppress per-item ublock/cookie warnings (those carry a locator). if (type === 'warning' && ublockWarningIgnored) return
// Batch summary warnings (locator = null) must always show.
if (type === 'warning' && ublockWarningIgnored && locator) return
const id = ++toastIdRef.current const id = ++toastIdRef.current
setToasts(prev => [...prev, { id, text, locator, type, headline }]) setToasts(prev => [...prev, { id, text, locator, type }])
}, [ublockWarningIgnored]) }, [ublockWarningIgnored])
const handleDismissToast = useCallback((id) => { const handleDismissToast = useCallback((id) => {
@ -256,7 +254,7 @@ export default function App() {
const handleIgnoreUblock = useCallback(() => { const handleIgnoreUblock = useCallback(() => {
sessionStorage.setItem('ublockWarningIgnored', 'true') sessionStorage.setItem('ublockWarningIgnored', 'true')
setUblockWarningIgnored(true) setUblockWarningIgnored(true)
setToasts(prev => prev.filter(t => !(t.type === 'warning' && t.locator))) setToasts(prev => prev.filter(t => t.type !== 'warning'))
}, []) }, [])
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>; if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;

View file

@ -371,18 +371,6 @@ export async function deleteOrphanBlobs(archiveId) {
return res.json() return res.json()
} }
// ── Re-archive ────────────────────────────────────────────────────────────────
export async function rearchiveEntry(archiveId, entryUid) {
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}/rearchive`, {
method: 'POST',
})
if (!res.ok) {
const body = await res.json().catch(() => ({}))
throw new Error(body.message || `rearchive failed: ${res.status}`)
}
return res.json() // { job_uid, status: 'pending' }
}
// ── Cookie rules ────────────────────────────────────────────────────────────── // ── Cookie rules ──────────────────────────────────────────────────────────────
export async function listCookieRules() { export async function listCookieRules() {

View file

@ -85,8 +85,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
const pollIntervals = useRef(new Map()) const pollIntervals = useRef(new Map())
// itemId debounce timeoutId for probe calls // itemId debounce timeoutId for probe calls
const probeTimers = useRef(new Map()) const probeTimers = useRef(new Map())
// batchId { total, archived, warnings, failed }; only populated for multi-URL submits
const batchRef = useRef(new Map())
// stable ref so probe callbacks always see the current archiveId // stable ref so probe callbacks always see the current archiveId
const archiveIdRef = useRef(archiveId) const archiveIdRef = useRef(archiveId)
useEffect(() => { archiveIdRef.current = archiveId }, [archiveId]) useEffect(() => { archiveIdRef.current = archiveId }, [archiveId])
@ -196,7 +194,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
} }
}, []) }, [])
function startPolling(itemId, jobUid, locator, aid, batchId = null) { function startPolling(itemId, jobUid, locator, aid) {
if (pollIntervals.current.has(jobUid)) return // already polling if (pollIntervals.current.has(jobUid)) return // already polling
const intervalId = setInterval(async () => { const intervalId = setInterval(async () => {
try { try {
@ -204,6 +202,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
if (updated.status === 'completed') { if (updated.status === 'completed') {
clearInterval(pollIntervals.current.get(jobUid)) clearInterval(pollIntervals.current.get(jobUid))
pollIntervals.current.delete(jobUid) pollIntervals.current.delete(jobUid)
// Show briefly then remove the row; if last row add a fresh one
setItems(prev => prev.map(it => it.id === itemId ? { ...it, status: 'completed' } : it)) setItems(prev => prev.map(it => it.id === itemId ? { ...it, status: 'completed' } : it))
setTimeout(() => { setTimeout(() => {
setItems(prev => { setItems(prev => {
@ -212,25 +211,13 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
}) })
}, 1400) }, 1400)
onCapturedRef.current() onCapturedRef.current()
// Warn if uBlock was requested but the extension wasn't available
try { try {
const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null
if (notes?.ublock_skipped || notes?.cookie_ext_skipped) { if (notes?.ublock_skipped || notes?.cookie_ext_skipped) {
const both = notes.ublock_skipped && notes.cookie_ext_skipped onToastRef.current(null, locator, 'warning')
const msg = both
? 'Captured without ad-blocking or cookie-consent extension. Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'
: notes.ublock_skipped
? 'Captured without ad-blocking. ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid.'
: 'Captured without cookie-consent extension. ARCHIVR_COOKIE_EXT is not set or the path is invalid.'
onToastRef.current(msg, locator, 'warning')
settleBatch(batchId, 'warning', locator)
} else {
if (!batchId) onToastRef.current(null, locator, 'success')
settleBatch(batchId, 'archived')
} }
} catch { } catch {}
if (!batchId) onToastRef.current(null, locator, 'success')
settleBatch(batchId, 'archived')
}
} else if (updated.status === 'failed') { } else if (updated.status === 'failed') {
clearInterval(pollIntervals.current.get(jobUid)) clearInterval(pollIntervals.current.get(jobUid))
pollIntervals.current.delete(jobUid) pollIntervals.current.delete(jobUid)
@ -239,7 +226,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
it.id === itemId ? { ...it, status: 'failed', error: errText } : it it.id === itemId ? { ...it, status: 'failed', error: errText } : it
)) ))
onToastRef.current(errText, locator) onToastRef.current(errText, locator)
settleBatch(batchId, 'failed', locator)
} }
// 'pending' / 'running': keep polling // 'pending' / 'running': keep polling
} catch (e) { } catch (e) {
@ -250,51 +236,14 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
it.id === itemId ? { ...it, status: 'failed', error: msg } : it it.id === itemId ? { ...it, status: 'failed', error: msg } : it
)) ))
onToastRef.current(msg, locator) onToastRef.current(msg, locator)
settleBatch(batchId, 'failed', locator)
} }
}, 500) }, 500)
pollIntervals.current.set(jobUid, intervalId) pollIntervals.current.set(jobUid, intervalId)
} }
// Increments the batch counter for the given outcome and emits a summary async function submitItem(item) {
// toast once all jobs in the batch have settled.
// 'warning' counts as archived (succeeded with caveats); locator recorded for detail text.
function settleBatch(batchId, outcome, locator = null) {
if (!batchId) return
const batch = batchRef.current.get(batchId)
if (!batch) return
if (outcome === 'archived' || outcome === 'warning') {
batch.archived++
if (outcome === 'warning') { batch.warnings++; if (locator) batch.warningLocators.push(locator) }
} else {
batch.failed++
if (locator) batch.failedLocators.push(locator)
}
if (batch.archived + batch.failed < batch.total) return
// All settled build headline + detail text, then emit summary and clean up.
batchRef.current.delete(batchId)
const { archived, warnings, failed, failedLocators, warningLocators } = batch
let headline
if (archived === 0) {
headline = `${failed} failed`
} else {
const archivedStr = warnings > 0
? `${archived} archived (${warnings} with warnings)`
: `${archived} archived`
headline = failed > 0 ? `${archivedStr}, ${failed} failed` : archivedStr
}
const type = archived === 0 ? 'error' : (failed > 0 || warnings > 0) ? 'warning' : 'success'
const parts = []
if (failedLocators.length > 0) parts.push(`Failed:\n${failedLocators.map(l => ` ${l}`).join('\n')}`)
if (warningLocators.length > 0) parts.push(`With warnings:\n${warningLocators.map(l => ` ${l}`).join('\n')}`)
const text = parts.length > 0 ? parts.join('\n') : null
onToastRef.current(text, null, type, headline)
}
async function submitItem(item, batchId = null) {
if (!item.locator.trim()) return if (!item.locator.trim()) return
const aid = archiveId const aid = archiveId // capture at submit time
const loc = item.locator.trim() const loc = item.locator.trim()
const qual = item.quality || 'best' const qual = item.quality || 'best'
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it)) setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
@ -304,27 +253,17 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
setItems(prev => prev.map(it => setItems(prev => prev.map(it =>
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
)) ))
startPolling(item.id, job.job_uid, loc, aid, batchId) startPolling(item.id, job.job_uid, loc, aid)
} catch (e) { } catch (e) {
const msg = e.message || 'Submission failed.' const msg = e.message || 'Submission failed.'
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'failed', error: msg } : it)) setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'failed', error: msg } : it))
onToastRef.current(msg, loc) onToastRef.current(msg, loc)
settleBatch(batchId, 'failed', loc)
} }
} }
function handleArchive() { function handleArchive() {
const toSubmit = items.filter(it => it.status === 'idle' && it.locator.trim()) const toSubmit = items.filter(it => it.status === 'idle' && it.locator.trim())
if (toSubmit.length === 0) return toSubmit.forEach(it => submitItem(it))
const batchId = toSubmit.length > 1
? (crypto.randomUUID?.() ?? `batch-${Date.now()}`)
: null
if (batchId) {
batchRef.current.set(batchId, { total: toSubmit.length, archived: 0, warnings: 0, failed: 0, failedLocators: [], warningLocators: [] })
}
toSubmit.forEach(it => submitItem(it, batchId))
dialogRef.current?.close()
} }
function addRow() { function addRow() {

View file

@ -1,5 +1,5 @@
import { useState, useEffect, useRef } from 'react' import { useState, useEffect, useRef } from 'react'
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry, rearchiveEntry, pollCaptureJob } from '../api' import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry } from '../api'
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils' import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' } const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
@ -21,14 +21,8 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
const titleCancelRef = useRef(false) const titleCancelRef = useRef(false)
const [editingTitle, setEditingTitle] = useState(false) const [editingTitle, setEditingTitle] = useState(false)
const [titleDraft, setTitleDraft] = useState('') const [titleDraft, setTitleDraft] = useState('')
const [rearchiveState, setRearchiveState] = useState('idle') // 'idle' | 'running' | 'done' | 'error'
const [rearchiveError, setRearchiveError] = useState('')
const rearchivePollRef = useRef(null)
useEffect(() => { useEffect(() => {
if (rearchivePollRef.current) { clearInterval(rearchivePollRef.current); rearchivePollRef.current = null }
setRearchiveState('idle')
setRearchiveError('')
if (!selectedEntry || !archiveId) { if (!selectedEntry || !archiveId) {
setDetail(null) setDetail(null)
setTags([]) setTags([])
@ -53,12 +47,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
}).catch(() => {}) }).catch(() => {})
}, [selectedEntry, archiveId]) }, [selectedEntry, archiveId])
useEffect(() => {
return () => {
clearInterval(rearchivePollRef.current)
}
}, [])
async function handleTitleSave() { async function handleTitleSave() {
const newTitle = titleDraft.trim() || null const newTitle = titleDraft.trim() || null
try { try {
@ -109,46 +97,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
} }
} }
async function handleRearchive() {
if (!selectedEntry || !archiveId || rearchiveState === 'running') return
setRearchiveState('running')
setRearchiveError('')
try {
const { job_uid } = await rearchiveEntry(archiveId, selectedEntry.entry_uid)
// Poll for job completion at 500ms same interval as CaptureDialog
rearchivePollRef.current = setInterval(async () => {
try {
const job = await pollCaptureJob(archiveId, job_uid)
if (job.status === 'completed') {
clearInterval(rearchivePollRef.current)
rearchivePollRef.current = null
setRearchiveState('done')
// Refresh entry detail so artifact list updates
const [det, tgs] = await Promise.all([
fetchEntryDetail(archiveId, selectedEntry.entry_uid),
fetchEntryTags(archiveId, selectedEntry.entry_uid),
])
setDetail(det)
setTags(tgs)
} else if (job.status === 'failed') {
clearInterval(rearchivePollRef.current)
rearchivePollRef.current = null
setRearchiveState('error')
setRearchiveError(job.error_text || 'Re-archive failed.')
}
} catch {
clearInterval(rearchivePollRef.current)
rearchivePollRef.current = null
setRearchiveState('error')
setRearchiveError('Network error while polling.')
}
}, 500)
} catch (e) {
setRearchiveState('error')
setRearchiveError(e.message || 'Failed to start re-archive.')
}
}
const metaRows = detail ? [ const metaRows = detail ? [
['Added', formatTimestamp(detail.summary.archived_at)], ['Added', formatTimestamp(detail.summary.archived_at)],
['Source', detail.summary.source_kind], ['Source', detail.summary.source_kind],
@ -296,25 +244,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
</div> </div>
)} )}
{detail && (detail.summary.entity_kind === 'tweet' || detail.summary.entity_kind === 'tweet_thread') && (
<div className="rail-section">
<div className="rail-section-heading">Actions</div>
<button
className="rail-rearchive-btn"
onClick={handleRearchive}
disabled={rearchiveState === 'running'}
>
{rearchiveState === 'running' ? 'Re-archiving\u2026' : 'Re-archive'}
</button>
{rearchiveState === 'done' && (
<p className="form-msg form-msg--ok" style={{ marginTop: '6px' }}>Re-archived successfully.</p>
)}
{rearchiveState === 'error' && (
<p className="form-msg form-msg--err" style={{ marginTop: '6px' }}>{rearchiveError}</p>
)}
</div>
)}
<div className="rail-delete-zone"> <div className="rail-delete-zone">
<button className="rail-delete-btn" onClick={handleDeleteEntry}> <button className="rail-delete-btn" onClick={handleDeleteEntry}>
Delete entry Delete entry

View file

@ -13,27 +13,9 @@ export default function ToastStack({ toasts, onDismiss, onIgnoreUblock }) {
) )
} }
// For URLs: show hostname//tail (drops protocol + middle path, keeps domain + identifier).
// For shorthands (yt:, x:, etc.): keep the tail since it's the whole identifier.
// CSS text-overflow:ellipsis on .toast-locator handles any remaining overflow.
function shortLocator(locator) {
if (!locator) return null
if (locator.length <= 52) return locator
try {
const { hostname, pathname, search } = new URL(locator)
const segments = pathname.split('/').filter(Boolean)
const tail = (segments[segments.length - 1] ?? '') + search
const candidate = tail ? `${hostname}/\u2026/${tail}` : hostname
return candidate.length <= 56 ? candidate : candidate.slice(0, 53) + '\u2026'
} catch {
return '\u2026' + locator.slice(-51)
}
}
function Toast({ toast, onDismiss, onIgnoreUblock }) { function Toast({ toast, onDismiss, onIgnoreUblock }) {
const [expanded, setExpanded] = useState(false) const [expanded, setExpanded] = useState(false)
const isWarning = toast.type === 'warning' const isWarning = toast.type === 'warning'
const isSuccess = toast.type === 'success'
// Auto-dismiss after TTL; paused while detail is expanded // Auto-dismiss after TTL; paused while detail is expanded
useEffect(() => { useEffect(() => {
@ -42,24 +24,9 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
return () => clearTimeout(timer) return () => clearTimeout(timer)
}, [expanded, toast.id, onDismiss]) }, [expanded, toast.id, onDismiss])
const short = shortLocator(toast.locator) const short = toast.locator
? (toast.locator.length > 48 ? toast.locator.slice(0, 45) + '\u2026' : toast.locator)
if (isSuccess) { : null
return (
<div className="toast toast--success" role="alert" aria-atomic="true">
<div className="toast-top">
<span className="toast-icon" aria-hidden="true"></span>
<div className="toast-body">
<span className="toast-headline">{toast.headline || 'Archived'}</span>
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
</div>
<div className="toast-btns">
<button type="button" className="toast-dismiss" onClick={() => onDismiss(toast.id)} aria-label="Dismiss">×</button>
</div>
</div>
</div>
)
}
if (isWarning) { if (isWarning) {
return ( return (
@ -67,29 +34,25 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
<div className="toast-top"> <div className="toast-top">
<span className="toast-icon" aria-hidden="true"></span> <span className="toast-icon" aria-hidden="true"></span>
<div className="toast-body"> <div className="toast-body">
<span className="toast-headline">{toast.headline || 'Archived with warnings'}</span> <span className="toast-headline">Ad-blocking unavailable</span>
{short && <span className="toast-locator" title={toast.locator}>{short}</span>} {short && <span className="toast-locator">{short}</span>}
</div> </div>
<div className="toast-btns"> <div className="toast-btns">
{toast.text && ( <button
<button type="button"
type="button" className="toast-view-btn"
className="toast-view-btn" onClick={() => setExpanded(v => !v)}
onClick={() => setExpanded(v => !v)} aria-expanded={expanded}
aria-expanded={expanded} >
> {expanded ? 'Hide' : 'Details'}
{expanded ? 'Hide' : 'Details'} </button>
</button> <button
)} type="button"
{toast.locator && ( className="toast-view-btn toast-ignore-btn"
<button onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }}
type="button" >
className="toast-view-btn toast-ignore-btn" Ignore
onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }} </button>
>
Ignore
</button>
)}
<button <button
type="button" type="button"
className="toast-dismiss" className="toast-dismiss"
@ -102,7 +65,7 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
</div> </div>
{expanded && ( {expanded && (
<p className="toast-warning-detail"> <p className="toast-warning-detail">
{toast.text || 'The page was captured but one or more browser extensions were unavailable (ad-blocking or cookie-consent). Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'} {toast.text || 'ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid. The page was captured without ad-blocking. Set ARCHIVR_UBLOCK_EXT to the unpacked uBlock Origin Lite extension directory to enable ad-blocking, or set ARCHIVR_UBLOCK=false to silence this warning.'}
</p> </p>
)} )}
</div> </div>
@ -114,8 +77,8 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
<div className="toast-top"> <div className="toast-top">
<span className="toast-icon" aria-hidden="true"></span> <span className="toast-icon" aria-hidden="true"></span>
<div className="toast-body"> <div className="toast-body">
<span className="toast-headline">{toast.headline || 'Capture failed'}</span> <span className="toast-headline">Capture failed</span>
{short && <span className="toast-locator" title={toast.locator}>{short}</span>} {short && <span className="toast-locator">{short}</span>}
</div> </div>
<div className="toast-btns"> <div className="toast-btns">
{toast.text && ( {toast.text && (

View file

@ -552,24 +552,6 @@ select {
border-color: var(--accent); border-color: var(--accent);
} }
.rail-rearchive-btn {
width: 100%;
padding: 6px 12px;
background: var(--surface-2, #2a2a2a);
color: var(--text, #e0e0e0);
border: 1px solid var(--border, #444);
border-radius: 4px;
cursor: pointer;
font-size: 13px;
}
.rail-rearchive-btn:hover:not(:disabled) {
background: var(--surface-3, #333);
}
.rail-rearchive-btn:disabled {
opacity: 0.6;
cursor: not-allowed;
}
/* ── Capture dialog ──────────────────────────────────────────────────────── */ /* ── Capture dialog ──────────────────────────────────────────────────────── */
.capture-dialog { .capture-dialog {
border: 1px solid var(--line); border: 1px solid var(--line);
@ -887,7 +869,6 @@ select {
} }
.toast--error { border-left: 3px solid var(--accent); } .toast--error { border-left: 3px solid var(--accent); }
.toast--warning { border-left: 3px solid #e8a000; } .toast--warning { border-left: 3px solid #e8a000; }
.toast--success { border-left: 3px solid #22a855; }
.toast-top { .toast-top {
display: flex; display: flex;
@ -902,8 +883,6 @@ select {
color: var(--accent); color: var(--accent);
margin-top: 2px; margin-top: 2px;
} }
.toast--success .toast-icon { color: #22a855; }
.toast--warning .toast-icon { color: #e8a000; }
.toast-body { .toast-body {
flex: 1; flex: 1;
min-width: 0; min-width: 0;