1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

feat(tweets): add re-archive button and fix thread-tweet orphan cleanup (#23)

Orphan cleanup bug: archiving x🧵A downloaded D/C/B/A JSONs and
media, but only registered artifacts for A. D/C/B files had no
entry_artifacts rows and were deleted as orphans.

Fix (staged scraper output, precise touched set):
- tweets::archive() stages all scraper output in temp/{ts}/tweet_stage/,
  validates, then renames JSONs to raw_tweets/. Return type changed from
  Result<bool> to Result<Vec<String>> (store-relative relpaths of every
  produced tweet JSON, i.e. the exact touched set).
- tweets::rearchive() (new): same staged approach but always runs the
  scraper. On scraper failure (tweet deleted/private), errors before
  touching raw_tweets/ so existing data is preserved.
- register_tweet_artifacts() (new private helper in capture.rs): registers
  every JSON in the touched set as a raw_tweet_json artifact, parses each
  for media blobs, registers those too. JSON read failure is a hard error
  with context, not a silent skip.
- record_tweet_entry() now accepts tweet_json_relpaths: &[String] and
  delegates artifact registration to register_tweet_artifacts().
- perform_capture() passes the returned vec from tweets::archive().

Re-archive feature:
- capture::perform_rearchive(): looks up entry by uid, validates
  tweet/tweet_thread, runs tweets::rearchive(), atomically swaps
  entry_artifacts in a DB transaction. archived_at, title, tags,
  collections are untouched.
- database: add get_entry_for_rearchive() and delete_entry_artifacts().
- POST /api/archives/:id/entries/:uid/rearchive: requires ROLE_USER,
  creates capture job, returns 202 + job_uid, runs perform_rearchive in
  spawn_blocking.
- Frontend: re-archive button in ContextRail for tweet/tweet_thread
  entries; polls job at 500ms; refreshes entry detail on success; shows
  error text on failure. Poll interval cleared before early-return on
  entry deselect to prevent stale updates.
This commit is contained in:
TheGeneralist 2026-07-11 16:14:58 +02:00 committed by GitHub
parent 03390362c5
commit 2779afee2d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 528 additions and 174 deletions

View file

@ -742,6 +742,52 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
} }
} }
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
fn register_tweet_artifacts(
conn: &rusqlite::Connection,
store_path: &Path,
entry_id: i64,
tweet_json_relpaths: &[String],
) -> Result<()> {
for relpath in tweet_json_relpaths {
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id,
artifact_role: "raw_tweet_json".to_string(),
storage_area: "raw_tweets".to_string(),
relpath: relpath.clone(),
blob_id: None,
logical_path: None,
metadata_json: None,
},
)?;
let json_path = store_path.join(relpath);
let json_str = fs::read_to_string(&json_path)
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
let raw_path = PathBuf::from(&raw_relpath);
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
let blob_id = database::upsert_blob(conn, &blob)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id,
artifact_role: role,
storage_area: "raw".to_string(),
relpath: raw_relpath,
blob_id: Some(blob_id),
logical_path: None,
metadata_json: None,
},
)?;
}
}
Ok(())
}
fn record_tweet_entry( fn record_tweet_entry(
conn: &rusqlite::Connection, conn: &rusqlite::Connection,
store_path: &Path, store_path: &Path,
@ -751,6 +797,7 @@ fn record_tweet_entry(
requested_locator: &str, requested_locator: &str,
source: Source, source: Source,
tweet_id: &str, tweet_id: &str,
tweet_json_relpaths: &[String],
) -> Result<database::ArchivedEntry> { ) -> Result<database::ArchivedEntry> {
debug_assert!(run.run_uid.starts_with("run_")); debug_assert!(run.run_uid.starts_with("run_"));
debug_assert!(item.item_uid.starts_with("item_")); debug_assert!(item.item_uid.starts_with("item_"));
@ -764,7 +811,7 @@ fn record_tweet_entry(
Some(&canonical_locator), Some(&canonical_locator),
&canonical_locator, &canonical_locator,
)?; )?;
// Read tweet JSON early to extract title before entry creation // Read the primary tweet JSON to extract title before entry creation.
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
let tweet_meta = tweet_metadata_from_json(&tweet_json); let tweet_meta = tweet_metadata_from_json(&tweet_json);
@ -794,36 +841,8 @@ fn record_tweet_entry(
)?; )?;
create_structured_root(store_path, &entry)?; create_structured_root(store_path, &entry)?;
database::add_entry_artifact( // Register all tweet JSONs and their blobs.
conn, register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?;
&database::NewArtifact {
entry_id: entry.id,
artifact_role: "raw_tweet_json".to_string(),
storage_area: "raw_tweets".to_string(),
relpath: path_to_store_string(&tweet_json_relpath),
blob_id: None,
logical_path: None,
metadata_json: None,
},
)?;
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
let raw_path = PathBuf::from(&raw_relpath);
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
let blob_id = database::upsert_blob(conn, &blob)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id: entry.id,
artifact_role: role,
storage_area: "raw".to_string(),
relpath: raw_relpath,
blob_id: Some(blob_id),
logical_path: None,
metadata_json: None,
},
)?;
}
database::complete_archive_run_item(conn, item.id, entry.id)?; database::complete_archive_run_item(conn, item.id, entry.id)?;
Ok(entry) Ok(entry)
@ -1184,7 +1203,7 @@ pub fn perform_capture(
&timestamp, &timestamp,
&tweet_cookies, &tweet_cookies,
) { ) {
Ok(_) => { Ok(tweet_json_relpaths) => {
let tweet_entry = record_tweet_entry( let tweet_entry = record_tweet_entry(
&conn, &conn,
store_path, store_path,
@ -1194,6 +1213,7 @@ pub fn perform_capture(
locator, locator,
source, source,
&tweet_id, &tweet_id,
&tweet_json_relpaths,
)?; )?;
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?; database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
@ -1362,6 +1382,107 @@ pub fn perform_capture(
}) })
} }
/// Result of a tweet re-archive operation.
#[derive(Debug, serde::Serialize)]
pub struct RearchiveResult {
/// One of: "completed", "not_a_tweet", "scraper_failed".
pub status: String,
/// Human-readable detail (empty string for "completed").
pub message: String,
}
/// Re-archives an existing tweet or tweet_thread entry in-place.
///
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
/// returns `scraper_failed` WITHOUT touching the existing data.
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
/// and their media blobs. Does not change archived_at, title, tags, or collections.
pub fn perform_rearchive(
archive_paths: &ArchivePaths,
entry_uid: &str,
config: &CaptureConfig,
) -> Result<RearchiveResult> {
let store_path = &archive_paths.store_path;
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
// Look up the entry.
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
None => {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: "entry not found".to_string(),
});
}
Some(e) => e,
};
// Only tweet / tweet_thread entries can be re-archived this way.
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: format!("entry is '{}', not a tweet", entry.entity_kind),
});
}
// Parse source_metadata_json for tweet_id and requested_locator.
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
.unwrap_or(serde_json::Value::Object(Default::default()));
let requested_locator = meta["requested_locator"]
.as_str()
.unwrap_or("")
.to_string();
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
if requested_locator.is_empty() || tweet_id.is_empty() {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
});
}
let is_thread = entry.entity_kind == "tweet_thread";
let timestamp = format!(
"{}-{}",
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
uuid::Uuid::new_v4().simple(),
);
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
// Run the scraper (staged). If this fails, existing data is untouched.
let tweet_json_relpaths = match downloader::tweets::rearchive(
&requested_locator,
is_thread,
store_path,
&timestamp,
&tweet_cookies,
) {
Ok(relpaths) => relpaths,
Err(e) => {
return Ok(RearchiveResult {
status: "scraper_failed".to_string(),
message: format!("{e:#}"),
});
}
};
// Atomically swap artifact rows: delete old, insert new.
{
let tx = conn.transaction()?;
database::delete_entry_artifacts(&tx, entry.id)?;
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
tx.commit()?;
}
database::refresh_entry_cached_bytes(&conn, entry.id)?;
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
Ok(RearchiveResult {
status: "completed".to_string(),
message: String::new(),
})
}
fn mime_for_font_ext(ext: &str) -> Option<String> { fn mime_for_font_ext(ext: &str) -> Option<String> {
match ext { match ext {
".woff2" => Some("font/woff2".to_string()), ".woff2" => Some("font/woff2".to_string()),
@ -1988,6 +2109,7 @@ mod tests {
"tweet:123", "tweet:123",
Source::Tweet, Source::Tweet,
"123", "123",
&["raw_tweets/tweet-123.json".to_string()],
) )
.unwrap(); .unwrap();
database::finish_archive_run(&conn, run.id).unwrap(); database::finish_archive_run(&conn, run.id).unwrap();

View file

@ -2391,6 +2391,43 @@ fn humanize_slug(slug: &str) -> String {
.join(" ") .join(" ")
} }
/// A minimal view of an archived entry needed for re-archiving.
pub struct EntryForRearchive {
pub id: i64,
pub entity_kind: String,
pub source_metadata_json: String,
}
/// Looks up an entry by its public UID for re-archive purposes.
pub fn get_entry_for_rearchive(
conn: &Connection,
entry_uid: &str,
) -> Result<Option<EntryForRearchive>> {
conn.query_row(
"SELECT id, entity_kind, source_metadata_json \
FROM archived_entries WHERE entry_uid = ?1",
[entry_uid],
|row| {
Ok(EntryForRearchive {
id: row.get(0)?,
entity_kind: row.get(1)?,
source_metadata_json: row.get(2)?,
})
},
)
.optional()
.map_err(anyhow::Error::from)
}
/// Deletes all `entry_artifacts` rows for the given entry.
/// Call within a transaction for atomicity with re-insertion.
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
Ok(conn.execute(
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
[entry_id],
)?)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

View file

@ -1,7 +1,7 @@
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use regex::Regex; use regex::Regex;
use std::{ use std::{
collections::{HashMap, HashSet}, collections::HashMap,
env, env,
ffi::OsString, ffi::OsString,
fs, fs,
@ -59,63 +59,39 @@ fn build_scraper_args(
args args
} }
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`). /// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs
/// /// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`).
/// Invokes the Python scraper, then moves all produced media assets into the /// The staging dir starts empty so all files found there are exactly the touched set.
/// content-addressed raw store and rewrites the JSON output to use the new fn run_scraper_staged(
/// store-relative paths. Returns `true` if new content was archived, `false` tweet_id: &str,
/// if the tweet was already present and `thread` is `false`.
///
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
pub fn archive(
path: &str,
thread: bool, thread: bool,
store_path: &Path, store_path: &Path,
timestamp: &str, timestamp: &str,
cookies: &HashMap<String, String>, cookies: &HashMap<String, String>,
) -> Result<bool> { ) -> Result<Vec<String>> {
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?; let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
// Output directory for Tweet JSON files.
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
// Scraper media-dir = staging_dir/media/
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
let staging_tweets_dir = staging_dir.join("raw_tweets");
fs::create_dir_all(&staging_tweets_dir)?;
// Final destination for tweet JSONs.
let output_dir = store_path.join("raw_tweets"); let output_dir = store_path.join("raw_tweets");
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
fs::create_dir_all(&output_dir)?; fs::create_dir_all(&output_dir)?;
fs::create_dir_all(&temp_dir)?;
// Path to the root - the to-be-archived tweet's JSON file.
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
if !thread && root_json.exists() {
return Ok(false);
}
let before = tweet_json_files(&output_dir)?;
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3")); let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER") let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
.map(PathBuf::from) .map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py")); .unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd); let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
// Credentials: only use cookie rules as Twitter credentials when the resolved
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
// requires. A global cookie rule for an unrelated site must not suppress the
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
// The temp file is written in the semicolon-delimited format the Python scraper
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
// subprocess returns so secrets are never left on disk on failure.
let temp_creds_path: Option<PathBuf>; let temp_creds_path: Option<PathBuf>;
let credentials_file: PathBuf; let credentials_file: PathBuf;
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
let has_twitter_cookies =
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
if has_twitter_cookies { if has_twitter_cookies {
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt"); let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
let creds_str = cookies let creds_str = cookies
.iter() .iter()
.map(|(k, v)| format!("{k}={v}")) .map(|(k, v)| format!("{k}={v}"))
@ -135,7 +111,7 @@ pub fn archive(
let mut f = std::fs::File::create(&cf) let mut f = std::fs::File::create(&cf)
.context("failed to write twitter credentials file")?; .context("failed to write twitter credentials file")?;
f.write_all(creds_str.as_bytes()) f.write_all(creds_str.as_bytes())
.context("failed to write twitter credentials file")?; .context("failed to write twitter credentials file")?
} }
temp_creds_path = Some(cf.clone()); temp_creds_path = Some(cf.clone());
credentials_file = cf; credentials_file = cf;
@ -156,28 +132,21 @@ pub fn archive(
} }
let mut cmd = Command::new(&python); let mut cmd = Command::new(&python);
cmd.current_dir(&temp_dir).arg(&scraper_path); // Run the scraper from staging_dir so relative paths in JSON resolve correctly.
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) { cmd.current_dir(&staging_dir).arg(&scraper_path);
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
cmd.arg(arg); cmd.arg(arg);
} }
// Hold the Result so we can delete the credentials file before propagating
// any error — including spawn failures.
let spawn_result = cmd.output().with_context(|| { let spawn_result = cmd.output().with_context(|| {
format!( format!("Failed to spawn tweet scraper at {}", scraper_path.display())
"Failed to spawn tweet scraper at {}",
scraper_path.display()
)
}); });
// Remove temp credentials file unconditionally — secrets must not persist.
if let Some(cf) = &temp_creds_path { if let Some(cf) = &temp_creds_path {
let _ = fs::remove_file(cf); let _ = fs::remove_file(cf);
} }
let output = spawn_result?; let output = spawn_result?;
if !output.status.success() { if !output.status.success() {
let _ = fs::remove_dir_all(&staging_dir);
let stderr = String::from_utf8_lossy(&output.stderr); let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
bail!( bail!(
@ -187,7 +156,9 @@ pub fn archive(
); );
} }
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
if !root_json.exists() { if !root_json.exists() {
let _ = fs::remove_dir_all(&staging_dir);
let stderr = String::from_utf8_lossy(&output.stderr); let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
bail!( bail!(
@ -198,13 +169,82 @@ pub fn archive(
); );
} }
cleanup_summary(&output_dir)?; // Remove the scraping_summary.json if the scraper left one.
let after = tweet_json_files(&output_dir)?; cleanup_summary(&staging_tweets_dir)?;
let new_jsons = new_tweet_jsons(&before, &after);
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
Ok(true) // Collect all tweet-*.json files from staging (this is the exact touched set).
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
})
.collect();
staged_jsons.sort();
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
// Move each staged JSON to its final destination, collecting store-relative relpaths.
let mut relpaths = Vec::with_capacity(staged_jsons.len());
for staged_path in &staged_jsons {
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
let dest = output_dir.join(filename);
fs::rename(staged_path, &dest)
.or_else(|_| {
fs::copy(staged_path, &dest).map(|_| ())?;
fs::remove_file(staged_path)
})
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
let rel = dest
.strip_prefix(store_path)
.context("dest tweet JSON is not under store_path")?;
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
}
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
let _ = fs::remove_dir_all(&staging_dir);
Ok(relpaths)
}
/// Archives a tweet (or full thread) identified by `path`.
///
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
/// For single tweets already on disk, returns the existing file path without re-downloading.
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
pub fn archive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<Vec<String>> {
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
if !thread && root_json.exists() {
// Already present; skip re-download, return the existing file for artifact registration.
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
}
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
}
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
/// Always runs the scraper; output is staged before replacing existing files.
/// Returns store-relative relpaths of all produced tweet JSON files.
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
pub fn rearchive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<Vec<String>> {
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
} }
/// Removes the `scraping_summary.json` file left by the scraper, if present. /// Removes the `scraping_summary.json` file left by the scraper, if present.
@ -216,33 +256,6 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
Ok(()) Ok(())
} }
/// Returns the set of `tweet-*.json` files present in `output_dir`.
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
let mut files = HashSet::new();
for entry in fs::read_dir(output_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_file()
&& path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
{
files.insert(path);
}
}
Ok(files)
}
/// Returns the sorted list of JSON files present in `after` but not in `before`.
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
files.sort();
files
}
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON. /// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
fn avatar_regex() -> &'static Regex { fn avatar_regex() -> &'static Regex {
@ -519,9 +532,9 @@ mod tests {
fs::write(&credentials, "ct0=test;auth_token=test").unwrap(); fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials); set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap(); let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
assert!(!archived); assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]);
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE"); remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
let _ = fs::remove_dir_all(store_path); let _ = fs::remove_dir_all(store_path);
@ -576,7 +589,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
{ {
"id": "$tweet_id", "id": "$tweet_id",
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] }, "entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" } "author": { "avatar_local_path": "$media_dir/avatars/author.jpg" }
} }
EOF EOF
"#, "#,
@ -592,16 +605,16 @@ EOF
set_test_env("ARCHIVR_TWEET_SCRAPER", &script); set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh"); set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap(); let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
let tweet_file = output_dir.join("tweet-123.json"); let tweet_file = output_dir.join("tweet-123.json");
let contents = fs::read_to_string(&tweet_file).unwrap(); let contents = fs::read_to_string(&tweet_file).unwrap();
assert!(archived); assert!(!relpaths.is_empty());
assert!(tweet_file.exists()); assert!(tweet_file.exists());
assert!(!output_dir.join("scraping_summary.json").exists()); assert!(!output_dir.join("scraping_summary.json").exists());
assert!(contents.contains(r#""avatar_local_path": "raw/"#)); assert!(contents.contains(r#""avatar_local_path": "raw/"#));
assert!(contents.contains(r#""local_path": "raw/"#)); assert!(contents.contains(r#""local_path": "raw/"#));
assert!(!store_path.join("temp").join("ts").exists()); assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists());
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE"); remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
remove_test_env("ARCHIVR_TWEET_SCRAPER"); remove_test_env("ARCHIVR_TWEET_SCRAPER");

View file

@ -221,6 +221,10 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index", "/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
get(serve_artifact), get(serve_artifact),
) )
.route(
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
post(rearchive_handler),
)
.route( .route(
"/api/archives/:archive_id/entries/:entry_uid/favicon", "/api/archives/:archive_id/entries/:entry_uid/favicon",
get(serve_entry_favicon), get(serve_entry_favicon),
@ -832,6 +836,83 @@ async fn get_capture_job_handler(
.ok_or_else(|| ApiError::not_found("capture job not found")) .ok_or_else(|| ApiError::not_found("capture job not found"))
} }
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
///
/// Re-archives an existing tweet or tweet_thread entry in-place:
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
///
/// Returns 202 immediately with a job_uid the client should poll via
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
async fn rearchive_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path((archive_id, entry_uid)): Path<(String, String)>,
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
auth_user.require_role(ROLE_USER)?;
let mounted = mounted_archive(&state, &archive_id)?;
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
.map_err(ApiError::from)?;
// Create a capture job record so the client can poll for completion.
let conn = database::open_or_initialize(&mounted.archive_path)?;
let job_uid = database::create_capture_job(&conn, &archive_id)?;
drop(conn);
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
Err(_) => vec![],
};
let capture_config = capture::CaptureConfig {
cookie_rules,
ublock_enabled: None,
cookie_ext_enabled: None,
reader_mode: false,
};
let job_uid_bg = job_uid.clone();
let archive_path = mounted.archive_path.clone();
tokio::task::spawn_blocking(move || {
let conn = match database::open_or_initialize(&archive_path) {
Ok(c) => c,
Err(e) => {
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
return;
}
};
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
Ok(result) => {
if result.status == "completed" {
database::update_capture_job_status(
&conn, &job_uid_bg, "completed", None, None, None,
).ok();
} else {
// "not_a_tweet" or "scraper_failed" — surface as a job failure
// so the client sees a meaningful error message.
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&result.message), None,
).ok();
}
}
Err(e) => {
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&format!("{e:#}")), None,
).ok();
}
}
});
Ok((
StatusCode::ACCEPTED,
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
))
}
/// `GET /api/archives/:id/captures/probe?locator=<url>` /// `GET /api/archives/:id/captures/probe?locator=<url>`
/// ///
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video /// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,8 +4,8 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title> <title>Archivr</title>
<script type="module" crossorigin src="/assets/index-BbGz9qlG.js"></script> <script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-D0Nzcia0.css"> <link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

View file

@ -371,6 +371,18 @@ export async function deleteOrphanBlobs(archiveId) {
return res.json() return res.json()
} }
// ── Re-archive ────────────────────────────────────────────────────────────────
export async function rearchiveEntry(archiveId, entryUid) {
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}/rearchive`, {
method: 'POST',
})
if (!res.ok) {
const body = await res.json().catch(() => ({}))
throw new Error(body.message || `rearchive failed: ${res.status}`)
}
return res.json() // { job_uid, status: 'pending' }
}
// ── Cookie rules ────────────────────────────────────────────────────────────── // ── Cookie rules ──────────────────────────────────────────────────────────────
export async function listCookieRules() { export async function listCookieRules() {

View file

@ -1,5 +1,5 @@
import { useState, useEffect, useRef } from 'react' import { useState, useEffect, useRef } from 'react'
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry } from '../api' import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry, rearchiveEntry, pollCaptureJob } from '../api'
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils' import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' } const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
@ -21,8 +21,14 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
const titleCancelRef = useRef(false) const titleCancelRef = useRef(false)
const [editingTitle, setEditingTitle] = useState(false) const [editingTitle, setEditingTitle] = useState(false)
const [titleDraft, setTitleDraft] = useState('') const [titleDraft, setTitleDraft] = useState('')
const [rearchiveState, setRearchiveState] = useState('idle') // 'idle' | 'running' | 'done' | 'error'
const [rearchiveError, setRearchiveError] = useState('')
const rearchivePollRef = useRef(null)
useEffect(() => { useEffect(() => {
if (rearchivePollRef.current) { clearInterval(rearchivePollRef.current); rearchivePollRef.current = null }
setRearchiveState('idle')
setRearchiveError('')
if (!selectedEntry || !archiveId) { if (!selectedEntry || !archiveId) {
setDetail(null) setDetail(null)
setTags([]) setTags([])
@ -47,6 +53,12 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
}).catch(() => {}) }).catch(() => {})
}, [selectedEntry, archiveId]) }, [selectedEntry, archiveId])
useEffect(() => {
return () => {
clearInterval(rearchivePollRef.current)
}
}, [])
async function handleTitleSave() { async function handleTitleSave() {
const newTitle = titleDraft.trim() || null const newTitle = titleDraft.trim() || null
try { try {
@ -97,6 +109,46 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
} }
} }
async function handleRearchive() {
if (!selectedEntry || !archiveId || rearchiveState === 'running') return
setRearchiveState('running')
setRearchiveError('')
try {
const { job_uid } = await rearchiveEntry(archiveId, selectedEntry.entry_uid)
// Poll for job completion at 500ms same interval as CaptureDialog
rearchivePollRef.current = setInterval(async () => {
try {
const job = await pollCaptureJob(archiveId, job_uid)
if (job.status === 'completed') {
clearInterval(rearchivePollRef.current)
rearchivePollRef.current = null
setRearchiveState('done')
// Refresh entry detail so artifact list updates
const [det, tgs] = await Promise.all([
fetchEntryDetail(archiveId, selectedEntry.entry_uid),
fetchEntryTags(archiveId, selectedEntry.entry_uid),
])
setDetail(det)
setTags(tgs)
} else if (job.status === 'failed') {
clearInterval(rearchivePollRef.current)
rearchivePollRef.current = null
setRearchiveState('error')
setRearchiveError(job.error_text || 'Re-archive failed.')
}
} catch {
clearInterval(rearchivePollRef.current)
rearchivePollRef.current = null
setRearchiveState('error')
setRearchiveError('Network error while polling.')
}
}, 500)
} catch (e) {
setRearchiveState('error')
setRearchiveError(e.message || 'Failed to start re-archive.')
}
}
const metaRows = detail ? [ const metaRows = detail ? [
['Added', formatTimestamp(detail.summary.archived_at)], ['Added', formatTimestamp(detail.summary.archived_at)],
['Source', detail.summary.source_kind], ['Source', detail.summary.source_kind],
@ -244,6 +296,25 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
</div> </div>
)} )}
{detail && (detail.summary.entity_kind === 'tweet' || detail.summary.entity_kind === 'tweet_thread') && (
<div className="rail-section">
<div className="rail-section-heading">Actions</div>
<button
className="rail-rearchive-btn"
onClick={handleRearchive}
disabled={rearchiveState === 'running'}
>
{rearchiveState === 'running' ? 'Re-archiving\u2026' : 'Re-archive'}
</button>
{rearchiveState === 'done' && (
<p className="form-msg form-msg--ok" style={{ marginTop: '6px' }}>Re-archived successfully.</p>
)}
{rearchiveState === 'error' && (
<p className="form-msg form-msg--err" style={{ marginTop: '6px' }}>{rearchiveError}</p>
)}
</div>
)}
<div className="rail-delete-zone"> <div className="rail-delete-zone">
<button className="rail-delete-btn" onClick={handleDeleteEntry}> <button className="rail-delete-btn" onClick={handleDeleteEntry}>
Delete entry Delete entry

View file

@ -552,6 +552,24 @@ select {
border-color: var(--accent); border-color: var(--accent);
} }
.rail-rearchive-btn {
width: 100%;
padding: 6px 12px;
background: var(--surface-2, #2a2a2a);
color: var(--text, #e0e0e0);
border: 1px solid var(--border, #444);
border-radius: 4px;
cursor: pointer;
font-size: 13px;
}
.rail-rearchive-btn:hover:not(:disabled) {
background: var(--surface-3, #333);
}
.rail-rearchive-btn:disabled {
opacity: 0.6;
cursor: not-allowed;
}
/* ── Capture dialog ──────────────────────────────────────────────────────── */ /* ── Capture dialog ──────────────────────────────────────────────────────── */
.capture-dialog { .capture-dialog {
border: 1px solid var(--line); border: 1px solid var(--line);