mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat(tweets): add re-archive button and fix thread-tweet orphan cleanup (#23)
Orphan cleanup bug: archiving x🧵A downloaded D/C/B/A JSONs and
media, but only registered artifacts for A. D/C/B files had no
entry_artifacts rows and were deleted as orphans.
Fix (staged scraper output, precise touched set):
- tweets::archive() stages all scraper output in temp/{ts}/tweet_stage/,
validates, then renames JSONs to raw_tweets/. Return type changed from
Result<bool> to Result<Vec<String>> (store-relative relpaths of every
produced tweet JSON, i.e. the exact touched set).
- tweets::rearchive() (new): same staged approach but always runs the
scraper. On scraper failure (tweet deleted/private), errors before
touching raw_tweets/ so existing data is preserved.
- register_tweet_artifacts() (new private helper in capture.rs): registers
every JSON in the touched set as a raw_tweet_json artifact, parses each
for media blobs, registers those too. JSON read failure is a hard error
with context, not a silent skip.
- record_tweet_entry() now accepts tweet_json_relpaths: &[String] and
delegates artifact registration to register_tweet_artifacts().
- perform_capture() passes the returned vec from tweets::archive().
Re-archive feature:
- capture::perform_rearchive(): looks up entry by uid, validates
tweet/tweet_thread, runs tweets::rearchive(), atomically swaps
entry_artifacts in a DB transaction. archived_at, title, tags,
collections are untouched.
- database: add get_entry_for_rearchive() and delete_entry_artifacts().
- POST /api/archives/:id/entries/:uid/rearchive: requires ROLE_USER,
creates capture job, returns 202 + job_uid, runs perform_rearchive in
spawn_blocking.
- Frontend: re-archive button in ContextRail for tweet/tweet_thread
entries; polls job at 500ms; refreshes entry detail on success; shows
error text on failure. Poll interval cleared before early-return on
entry deselect to prevent stale updates.
This commit is contained in:
parent
03390362c5
commit
2779afee2d
12 changed files with 528 additions and 174 deletions
|
|
@ -742,6 +742,52 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
|
||||||
|
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
|
||||||
|
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
|
||||||
|
fn register_tweet_artifacts(
|
||||||
|
conn: &rusqlite::Connection,
|
||||||
|
store_path: &Path,
|
||||||
|
entry_id: i64,
|
||||||
|
tweet_json_relpaths: &[String],
|
||||||
|
) -> Result<()> {
|
||||||
|
for relpath in tweet_json_relpaths {
|
||||||
|
database::add_entry_artifact(
|
||||||
|
conn,
|
||||||
|
&database::NewArtifact {
|
||||||
|
entry_id,
|
||||||
|
artifact_role: "raw_tweet_json".to_string(),
|
||||||
|
storage_area: "raw_tweets".to_string(),
|
||||||
|
relpath: relpath.clone(),
|
||||||
|
blob_id: None,
|
||||||
|
logical_path: None,
|
||||||
|
metadata_json: None,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
let json_path = store_path.join(relpath);
|
||||||
|
let json_str = fs::read_to_string(&json_path)
|
||||||
|
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
|
||||||
|
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
|
||||||
|
let raw_path = PathBuf::from(&raw_relpath);
|
||||||
|
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
||||||
|
let blob_id = database::upsert_blob(conn, &blob)?;
|
||||||
|
database::add_entry_artifact(
|
||||||
|
conn,
|
||||||
|
&database::NewArtifact {
|
||||||
|
entry_id,
|
||||||
|
artifact_role: role,
|
||||||
|
storage_area: "raw".to_string(),
|
||||||
|
relpath: raw_relpath,
|
||||||
|
blob_id: Some(blob_id),
|
||||||
|
logical_path: None,
|
||||||
|
metadata_json: None,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn record_tweet_entry(
|
fn record_tweet_entry(
|
||||||
conn: &rusqlite::Connection,
|
conn: &rusqlite::Connection,
|
||||||
store_path: &Path,
|
store_path: &Path,
|
||||||
|
|
@ -751,6 +797,7 @@ fn record_tweet_entry(
|
||||||
requested_locator: &str,
|
requested_locator: &str,
|
||||||
source: Source,
|
source: Source,
|
||||||
tweet_id: &str,
|
tweet_id: &str,
|
||||||
|
tweet_json_relpaths: &[String],
|
||||||
) -> Result<database::ArchivedEntry> {
|
) -> Result<database::ArchivedEntry> {
|
||||||
debug_assert!(run.run_uid.starts_with("run_"));
|
debug_assert!(run.run_uid.starts_with("run_"));
|
||||||
debug_assert!(item.item_uid.starts_with("item_"));
|
debug_assert!(item.item_uid.starts_with("item_"));
|
||||||
|
|
@ -764,7 +811,7 @@ fn record_tweet_entry(
|
||||||
Some(&canonical_locator),
|
Some(&canonical_locator),
|
||||||
&canonical_locator,
|
&canonical_locator,
|
||||||
)?;
|
)?;
|
||||||
// Read tweet JSON early to extract title before entry creation
|
// Read the primary tweet JSON to extract title before entry creation.
|
||||||
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||||
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
|
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
|
||||||
let tweet_meta = tweet_metadata_from_json(&tweet_json);
|
let tweet_meta = tweet_metadata_from_json(&tweet_json);
|
||||||
|
|
@ -794,36 +841,8 @@ fn record_tweet_entry(
|
||||||
)?;
|
)?;
|
||||||
create_structured_root(store_path, &entry)?;
|
create_structured_root(store_path, &entry)?;
|
||||||
|
|
||||||
database::add_entry_artifact(
|
// Register all tweet JSONs and their blobs.
|
||||||
conn,
|
register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?;
|
||||||
&database::NewArtifact {
|
|
||||||
entry_id: entry.id,
|
|
||||||
artifact_role: "raw_tweet_json".to_string(),
|
|
||||||
storage_area: "raw_tweets".to_string(),
|
|
||||||
relpath: path_to_store_string(&tweet_json_relpath),
|
|
||||||
blob_id: None,
|
|
||||||
logical_path: None,
|
|
||||||
metadata_json: None,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
|
|
||||||
let raw_path = PathBuf::from(&raw_relpath);
|
|
||||||
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
|
||||||
let blob_id = database::upsert_blob(conn, &blob)?;
|
|
||||||
database::add_entry_artifact(
|
|
||||||
conn,
|
|
||||||
&database::NewArtifact {
|
|
||||||
entry_id: entry.id,
|
|
||||||
artifact_role: role,
|
|
||||||
storage_area: "raw".to_string(),
|
|
||||||
relpath: raw_relpath,
|
|
||||||
blob_id: Some(blob_id),
|
|
||||||
logical_path: None,
|
|
||||||
metadata_json: None,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
database::complete_archive_run_item(conn, item.id, entry.id)?;
|
database::complete_archive_run_item(conn, item.id, entry.id)?;
|
||||||
Ok(entry)
|
Ok(entry)
|
||||||
|
|
@ -1184,7 +1203,7 @@ pub fn perform_capture(
|
||||||
×tamp,
|
×tamp,
|
||||||
&tweet_cookies,
|
&tweet_cookies,
|
||||||
) {
|
) {
|
||||||
Ok(_) => {
|
Ok(tweet_json_relpaths) => {
|
||||||
let tweet_entry = record_tweet_entry(
|
let tweet_entry = record_tweet_entry(
|
||||||
&conn,
|
&conn,
|
||||||
store_path,
|
store_path,
|
||||||
|
|
@ -1194,6 +1213,7 @@ pub fn perform_capture(
|
||||||
locator,
|
locator,
|
||||||
source,
|
source,
|
||||||
&tweet_id,
|
&tweet_id,
|
||||||
|
&tweet_json_relpaths,
|
||||||
)?;
|
)?;
|
||||||
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
|
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
|
||||||
database::finish_archive_run(&conn, run.id)?;
|
database::finish_archive_run(&conn, run.id)?;
|
||||||
|
|
@ -1362,6 +1382,107 @@ pub fn perform_capture(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Result of a tweet re-archive operation.
|
||||||
|
#[derive(Debug, serde::Serialize)]
|
||||||
|
pub struct RearchiveResult {
|
||||||
|
/// One of: "completed", "not_a_tweet", "scraper_failed".
|
||||||
|
pub status: String,
|
||||||
|
/// Human-readable detail (empty string for "completed").
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-archives an existing tweet or tweet_thread entry in-place.
|
||||||
|
///
|
||||||
|
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
|
||||||
|
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
|
||||||
|
/// returns `scraper_failed` WITHOUT touching the existing data.
|
||||||
|
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
|
||||||
|
/// and their media blobs. Does not change archived_at, title, tags, or collections.
|
||||||
|
pub fn perform_rearchive(
|
||||||
|
archive_paths: &ArchivePaths,
|
||||||
|
entry_uid: &str,
|
||||||
|
config: &CaptureConfig,
|
||||||
|
) -> Result<RearchiveResult> {
|
||||||
|
let store_path = &archive_paths.store_path;
|
||||||
|
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
|
||||||
|
|
||||||
|
// Look up the entry.
|
||||||
|
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
|
||||||
|
None => {
|
||||||
|
return Ok(RearchiveResult {
|
||||||
|
status: "not_a_tweet".to_string(),
|
||||||
|
message: "entry not found".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Some(e) => e,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Only tweet / tweet_thread entries can be re-archived this way.
|
||||||
|
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
|
||||||
|
return Ok(RearchiveResult {
|
||||||
|
status: "not_a_tweet".to_string(),
|
||||||
|
message: format!("entry is '{}', not a tweet", entry.entity_kind),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse source_metadata_json for tweet_id and requested_locator.
|
||||||
|
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
|
||||||
|
.unwrap_or(serde_json::Value::Object(Default::default()));
|
||||||
|
let requested_locator = meta["requested_locator"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
|
||||||
|
if requested_locator.is_empty() || tweet_id.is_empty() {
|
||||||
|
return Ok(RearchiveResult {
|
||||||
|
status: "not_a_tweet".to_string(),
|
||||||
|
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let is_thread = entry.entity_kind == "tweet_thread";
|
||||||
|
let timestamp = format!(
|
||||||
|
"{}-{}",
|
||||||
|
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
|
||||||
|
uuid::Uuid::new_v4().simple(),
|
||||||
|
);
|
||||||
|
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
|
||||||
|
|
||||||
|
// Run the scraper (staged). If this fails, existing data is untouched.
|
||||||
|
let tweet_json_relpaths = match downloader::tweets::rearchive(
|
||||||
|
&requested_locator,
|
||||||
|
is_thread,
|
||||||
|
store_path,
|
||||||
|
×tamp,
|
||||||
|
&tweet_cookies,
|
||||||
|
) {
|
||||||
|
Ok(relpaths) => relpaths,
|
||||||
|
Err(e) => {
|
||||||
|
return Ok(RearchiveResult {
|
||||||
|
status: "scraper_failed".to_string(),
|
||||||
|
message: format!("{e:#}"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Atomically swap artifact rows: delete old, insert new.
|
||||||
|
{
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
database::delete_entry_artifacts(&tx, entry.id)?;
|
||||||
|
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
|
||||||
|
tx.commit()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
database::refresh_entry_cached_bytes(&conn, entry.id)?;
|
||||||
|
|
||||||
|
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
|
||||||
|
|
||||||
|
Ok(RearchiveResult {
|
||||||
|
status: "completed".to_string(),
|
||||||
|
message: String::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn mime_for_font_ext(ext: &str) -> Option<String> {
|
fn mime_for_font_ext(ext: &str) -> Option<String> {
|
||||||
match ext {
|
match ext {
|
||||||
".woff2" => Some("font/woff2".to_string()),
|
".woff2" => Some("font/woff2".to_string()),
|
||||||
|
|
@ -1988,6 +2109,7 @@ mod tests {
|
||||||
"tweet:123",
|
"tweet:123",
|
||||||
Source::Tweet,
|
Source::Tweet,
|
||||||
"123",
|
"123",
|
||||||
|
&["raw_tweets/tweet-123.json".to_string()],
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
database::finish_archive_run(&conn, run.id).unwrap();
|
database::finish_archive_run(&conn, run.id).unwrap();
|
||||||
|
|
|
||||||
|
|
@ -2391,6 +2391,43 @@ fn humanize_slug(slug: &str) -> String {
|
||||||
.join(" ")
|
.join(" ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A minimal view of an archived entry needed for re-archiving.
|
||||||
|
pub struct EntryForRearchive {
|
||||||
|
pub id: i64,
|
||||||
|
pub entity_kind: String,
|
||||||
|
pub source_metadata_json: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Looks up an entry by its public UID for re-archive purposes.
|
||||||
|
pub fn get_entry_for_rearchive(
|
||||||
|
conn: &Connection,
|
||||||
|
entry_uid: &str,
|
||||||
|
) -> Result<Option<EntryForRearchive>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, entity_kind, source_metadata_json \
|
||||||
|
FROM archived_entries WHERE entry_uid = ?1",
|
||||||
|
[entry_uid],
|
||||||
|
|row| {
|
||||||
|
Ok(EntryForRearchive {
|
||||||
|
id: row.get(0)?,
|
||||||
|
entity_kind: row.get(1)?,
|
||||||
|
source_metadata_json: row.get(2)?,
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
.map_err(anyhow::Error::from)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deletes all `entry_artifacts` rows for the given entry.
|
||||||
|
/// Call within a transaction for atomicity with re-insertion.
|
||||||
|
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
|
||||||
|
Ok(conn.execute(
|
||||||
|
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
|
||||||
|
[entry_id],
|
||||||
|
)?)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet},
|
collections::HashMap,
|
||||||
env,
|
env,
|
||||||
ffi::OsString,
|
ffi::OsString,
|
||||||
fs,
|
fs,
|
||||||
|
|
@ -59,63 +59,39 @@ fn build_scraper_args(
|
||||||
args
|
args
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
|
/// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs
|
||||||
///
|
/// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`).
|
||||||
/// Invokes the Python scraper, then moves all produced media assets into the
|
/// The staging dir starts empty so all files found there are exactly the touched set.
|
||||||
/// content-addressed raw store and rewrites the JSON output to use the new
|
fn run_scraper_staged(
|
||||||
/// store-relative paths. Returns `true` if new content was archived, `false`
|
tweet_id: &str,
|
||||||
/// if the tweet was already present and `thread` is `false`.
|
|
||||||
///
|
|
||||||
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
|
||||||
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
|
||||||
pub fn archive(
|
|
||||||
path: &str,
|
|
||||||
thread: bool,
|
thread: bool,
|
||||||
store_path: &Path,
|
store_path: &Path,
|
||||||
timestamp: &str,
|
timestamp: &str,
|
||||||
cookies: &HashMap<String, String>,
|
cookies: &HashMap<String, String>,
|
||||||
) -> Result<bool> {
|
) -> Result<Vec<String>> {
|
||||||
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
||||||
// Output directory for Tweet JSON files.
|
|
||||||
|
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
|
||||||
|
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
|
||||||
|
// Scraper media-dir = staging_dir/media/
|
||||||
|
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
|
||||||
|
let staging_tweets_dir = staging_dir.join("raw_tweets");
|
||||||
|
fs::create_dir_all(&staging_tweets_dir)?;
|
||||||
|
|
||||||
|
// Final destination for tweet JSONs.
|
||||||
let output_dir = store_path.join("raw_tweets");
|
let output_dir = store_path.join("raw_tweets");
|
||||||
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
|
|
||||||
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
|
|
||||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
|
||||||
|
|
||||||
fs::create_dir_all(&output_dir)?;
|
fs::create_dir_all(&output_dir)?;
|
||||||
fs::create_dir_all(&temp_dir)?;
|
|
||||||
|
|
||||||
// Path to the root - the to-be-archived tweet's JSON file.
|
|
||||||
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
|
|
||||||
if !thread && root_json.exists() {
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
let before = tweet_json_files(&output_dir)?;
|
|
||||||
|
|
||||||
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
||||||
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
||||||
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
||||||
|
|
||||||
// Credentials: only use cookie rules as Twitter credentials when the resolved
|
|
||||||
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
|
|
||||||
// requires. A global cookie rule for an unrelated site must not suppress the
|
|
||||||
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
|
|
||||||
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
|
|
||||||
// The temp file is written in the semicolon-delimited format the Python scraper
|
|
||||||
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
|
|
||||||
// subprocess returns so secrets are never left on disk on failure.
|
|
||||||
let temp_creds_path: Option<PathBuf>;
|
let temp_creds_path: Option<PathBuf>;
|
||||||
let credentials_file: PathBuf;
|
let credentials_file: PathBuf;
|
||||||
|
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||||
let has_twitter_cookies =
|
|
||||||
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
|
||||||
|
|
||||||
if has_twitter_cookies {
|
if has_twitter_cookies {
|
||||||
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
||||||
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
|
|
||||||
let creds_str = cookies
|
let creds_str = cookies
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(k, v)| format!("{k}={v}"))
|
.map(|(k, v)| format!("{k}={v}"))
|
||||||
|
|
@ -135,7 +111,7 @@ pub fn archive(
|
||||||
let mut f = std::fs::File::create(&cf)
|
let mut f = std::fs::File::create(&cf)
|
||||||
.context("failed to write twitter credentials file")?;
|
.context("failed to write twitter credentials file")?;
|
||||||
f.write_all(creds_str.as_bytes())
|
f.write_all(creds_str.as_bytes())
|
||||||
.context("failed to write twitter credentials file")?;
|
.context("failed to write twitter credentials file")?
|
||||||
}
|
}
|
||||||
temp_creds_path = Some(cf.clone());
|
temp_creds_path = Some(cf.clone());
|
||||||
credentials_file = cf;
|
credentials_file = cf;
|
||||||
|
|
@ -156,28 +132,21 @@ pub fn archive(
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut cmd = Command::new(&python);
|
let mut cmd = Command::new(&python);
|
||||||
cmd.current_dir(&temp_dir).arg(&scraper_path);
|
// Run the scraper from staging_dir so relative paths in JSON resolve correctly.
|
||||||
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
|
cmd.current_dir(&staging_dir).arg(&scraper_path);
|
||||||
|
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
|
||||||
cmd.arg(arg);
|
cmd.arg(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hold the Result so we can delete the credentials file before propagating
|
|
||||||
// any error — including spawn failures.
|
|
||||||
let spawn_result = cmd.output().with_context(|| {
|
let spawn_result = cmd.output().with_context(|| {
|
||||||
format!(
|
format!("Failed to spawn tweet scraper at {}", scraper_path.display())
|
||||||
"Failed to spawn tweet scraper at {}",
|
|
||||||
scraper_path.display()
|
|
||||||
)
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// Remove temp credentials file unconditionally — secrets must not persist.
|
|
||||||
if let Some(cf) = &temp_creds_path {
|
if let Some(cf) = &temp_creds_path {
|
||||||
let _ = fs::remove_file(cf);
|
let _ = fs::remove_file(cf);
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = spawn_result?;
|
let output = spawn_result?;
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
|
let _ = fs::remove_dir_all(&staging_dir);
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
bail!(
|
bail!(
|
||||||
|
|
@ -187,7 +156,9 @@ pub fn archive(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
|
||||||
if !root_json.exists() {
|
if !root_json.exists() {
|
||||||
|
let _ = fs::remove_dir_all(&staging_dir);
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
bail!(
|
bail!(
|
||||||
|
|
@ -198,13 +169,82 @@ pub fn archive(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup_summary(&output_dir)?;
|
// Remove the scraping_summary.json if the scraper left one.
|
||||||
let after = tweet_json_files(&output_dir)?;
|
cleanup_summary(&staging_tweets_dir)?;
|
||||||
let new_jsons = new_tweet_jsons(&before, &after);
|
|
||||||
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
|
|
||||||
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
|
|
||||||
|
|
||||||
Ok(true)
|
// Collect all tweet-*.json files from staging (this is the exact touched set).
|
||||||
|
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
.map(|e| e.path())
|
||||||
|
.filter(|p| {
|
||||||
|
p.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
staged_jsons.sort();
|
||||||
|
|
||||||
|
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
|
||||||
|
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
|
||||||
|
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
|
||||||
|
|
||||||
|
// Move each staged JSON to its final destination, collecting store-relative relpaths.
|
||||||
|
let mut relpaths = Vec::with_capacity(staged_jsons.len());
|
||||||
|
for staged_path in &staged_jsons {
|
||||||
|
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
|
||||||
|
let dest = output_dir.join(filename);
|
||||||
|
fs::rename(staged_path, &dest)
|
||||||
|
.or_else(|_| {
|
||||||
|
fs::copy(staged_path, &dest).map(|_| ())?;
|
||||||
|
fs::remove_file(staged_path)
|
||||||
|
})
|
||||||
|
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
|
||||||
|
let rel = dest
|
||||||
|
.strip_prefix(store_path)
|
||||||
|
.context("dest tweet JSON is not under store_path")?;
|
||||||
|
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
|
||||||
|
let _ = fs::remove_dir_all(&staging_dir);
|
||||||
|
|
||||||
|
Ok(relpaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Archives a tweet (or full thread) identified by `path`.
|
||||||
|
///
|
||||||
|
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
|
||||||
|
/// For single tweets already on disk, returns the existing file path without re-downloading.
|
||||||
|
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
|
||||||
|
pub fn archive(
|
||||||
|
path: &str,
|
||||||
|
thread: bool,
|
||||||
|
store_path: &Path,
|
||||||
|
timestamp: &str,
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
|
) -> Result<Vec<String>> {
|
||||||
|
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||||
|
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||||
|
if !thread && root_json.exists() {
|
||||||
|
// Already present; skip re-download, return the existing file for artifact registration.
|
||||||
|
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
|
||||||
|
}
|
||||||
|
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
|
||||||
|
/// Always runs the scraper; output is staged before replacing existing files.
|
||||||
|
/// Returns store-relative relpaths of all produced tweet JSON files.
|
||||||
|
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
|
||||||
|
pub fn rearchive(
|
||||||
|
path: &str,
|
||||||
|
thread: bool,
|
||||||
|
store_path: &Path,
|
||||||
|
timestamp: &str,
|
||||||
|
cookies: &HashMap<String, String>,
|
||||||
|
) -> Result<Vec<String>> {
|
||||||
|
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||||
|
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
||||||
|
|
@ -216,33 +256,6 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the set of `tweet-*.json` files present in `output_dir`.
|
|
||||||
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
|
|
||||||
let mut files = HashSet::new();
|
|
||||||
|
|
||||||
for entry in fs::read_dir(output_dir)? {
|
|
||||||
let entry = entry?;
|
|
||||||
let path = entry.path();
|
|
||||||
|
|
||||||
if path.is_file()
|
|
||||||
&& path
|
|
||||||
.file_name()
|
|
||||||
.and_then(|name| name.to_str())
|
|
||||||
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
|
|
||||||
{
|
|
||||||
files.insert(path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(files)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns the sorted list of JSON files present in `after` but not in `before`.
|
|
||||||
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
|
|
||||||
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
|
|
||||||
files.sort();
|
|
||||||
files
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
||||||
fn avatar_regex() -> &'static Regex {
|
fn avatar_regex() -> &'static Regex {
|
||||||
|
|
@ -519,9 +532,9 @@ mod tests {
|
||||||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||||
|
|
||||||
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||||
|
|
||||||
assert!(!archived);
|
assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]);
|
||||||
|
|
||||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||||
let _ = fs::remove_dir_all(store_path);
|
let _ = fs::remove_dir_all(store_path);
|
||||||
|
|
@ -576,7 +589,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
|
||||||
{
|
{
|
||||||
"id": "$tweet_id",
|
"id": "$tweet_id",
|
||||||
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
||||||
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
|
"author": { "avatar_local_path": "$media_dir/avatars/author.jpg" }
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
"#,
|
"#,
|
||||||
|
|
@ -592,16 +605,16 @@ EOF
|
||||||
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
||||||
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
||||||
|
|
||||||
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||||
let tweet_file = output_dir.join("tweet-123.json");
|
let tweet_file = output_dir.join("tweet-123.json");
|
||||||
let contents = fs::read_to_string(&tweet_file).unwrap();
|
let contents = fs::read_to_string(&tweet_file).unwrap();
|
||||||
|
|
||||||
assert!(archived);
|
assert!(!relpaths.is_empty());
|
||||||
assert!(tweet_file.exists());
|
assert!(tweet_file.exists());
|
||||||
assert!(!output_dir.join("scraping_summary.json").exists());
|
assert!(!output_dir.join("scraping_summary.json").exists());
|
||||||
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
||||||
assert!(contents.contains(r#""local_path": "raw/"#));
|
assert!(contents.contains(r#""local_path": "raw/"#));
|
||||||
assert!(!store_path.join("temp").join("ts").exists());
|
assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists());
|
||||||
|
|
||||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||||
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
||||||
|
|
|
||||||
|
|
@ -221,6 +221,10 @@ pub fn app_with_state(state: AppState) -> Router {
|
||||||
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
||||||
get(serve_artifact),
|
get(serve_artifact),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
|
||||||
|
post(rearchive_handler),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
||||||
get(serve_entry_favicon),
|
get(serve_entry_favicon),
|
||||||
|
|
@ -832,6 +836,83 @@ async fn get_capture_job_handler(
|
||||||
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
|
||||||
|
///
|
||||||
|
/// Re-archives an existing tweet or tweet_thread entry in-place:
|
||||||
|
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
|
||||||
|
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
|
||||||
|
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
|
||||||
|
///
|
||||||
|
/// Returns 202 immediately with a job_uid the client should poll via
|
||||||
|
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
|
||||||
|
async fn rearchive_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
auth_user: AuthUser,
|
||||||
|
Path((archive_id, entry_uid)): Path<(String, String)>,
|
||||||
|
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
|
||||||
|
auth_user.require_role(ROLE_USER)?;
|
||||||
|
let mounted = mounted_archive(&state, &archive_id)?;
|
||||||
|
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
|
||||||
|
.map_err(ApiError::from)?;
|
||||||
|
|
||||||
|
// Create a capture job record so the client can poll for completion.
|
||||||
|
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||||
|
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
|
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
|
||||||
|
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
|
||||||
|
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
||||||
|
Err(_) => vec![],
|
||||||
|
};
|
||||||
|
let capture_config = capture::CaptureConfig {
|
||||||
|
cookie_rules,
|
||||||
|
ublock_enabled: None,
|
||||||
|
cookie_ext_enabled: None,
|
||||||
|
reader_mode: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let job_uid_bg = job_uid.clone();
|
||||||
|
let archive_path = mounted.archive_path.clone();
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let conn = match database::open_or_initialize(&archive_path) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
|
||||||
|
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
|
||||||
|
Ok(result) => {
|
||||||
|
if result.status == "completed" {
|
||||||
|
database::update_capture_job_status(
|
||||||
|
&conn, &job_uid_bg, "completed", None, None, None,
|
||||||
|
).ok();
|
||||||
|
} else {
|
||||||
|
// "not_a_tweet" or "scraper_failed" — surface as a job failure
|
||||||
|
// so the client sees a meaningful error message.
|
||||||
|
database::update_capture_job_status(
|
||||||
|
&conn, &job_uid_bg, "failed",
|
||||||
|
None, Some(&result.message), None,
|
||||||
|
).ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
database::update_capture_job_status(
|
||||||
|
&conn, &job_uid_bg, "failed",
|
||||||
|
None, Some(&format!("{e:#}")), None,
|
||||||
|
).ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
StatusCode::ACCEPTED,
|
||||||
|
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
||||||
///
|
///
|
||||||
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
45
crates/archivr-server/static/assets/index-DiHAMIVH.js
Normal file
45
crates/archivr-server/static/assets/index-DiHAMIVH.js
Normal file
File diff suppressed because one or more lines are too long
1
crates/archivr-server/static/assets/index-DwI288Sc.css
Normal file
1
crates/archivr-server/static/assets/index-DwI288Sc.css
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>Archivr</title>
|
<title>Archivr</title>
|
||||||
<script type="module" crossorigin src="/assets/index-BbGz9qlG.js"></script>
|
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-D0Nzcia0.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div id="root"></div>
|
<div id="root"></div>
|
||||||
|
|
|
||||||
|
|
@ -371,6 +371,18 @@ export async function deleteOrphanBlobs(archiveId) {
|
||||||
return res.json()
|
return res.json()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Re-archive ────────────────────────────────────────────────────────────────
|
||||||
|
export async function rearchiveEntry(archiveId, entryUid) {
|
||||||
|
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}/rearchive`, {
|
||||||
|
method: 'POST',
|
||||||
|
})
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.json().catch(() => ({}))
|
||||||
|
throw new Error(body.message || `rearchive failed: ${res.status}`)
|
||||||
|
}
|
||||||
|
return res.json() // { job_uid, status: 'pending' }
|
||||||
|
}
|
||||||
|
|
||||||
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
export async function listCookieRules() {
|
export async function listCookieRules() {
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { useState, useEffect, useRef } from 'react'
|
import { useState, useEffect, useRef } from 'react'
|
||||||
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry } from '../api'
|
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry, rearchiveEntry, pollCaptureJob } from '../api'
|
||||||
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
|
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
|
||||||
|
|
||||||
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
|
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
|
||||||
|
|
@ -21,8 +21,14 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
const titleCancelRef = useRef(false)
|
const titleCancelRef = useRef(false)
|
||||||
const [editingTitle, setEditingTitle] = useState(false)
|
const [editingTitle, setEditingTitle] = useState(false)
|
||||||
const [titleDraft, setTitleDraft] = useState('')
|
const [titleDraft, setTitleDraft] = useState('')
|
||||||
|
const [rearchiveState, setRearchiveState] = useState('idle') // 'idle' | 'running' | 'done' | 'error'
|
||||||
|
const [rearchiveError, setRearchiveError] = useState('')
|
||||||
|
const rearchivePollRef = useRef(null)
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
if (rearchivePollRef.current) { clearInterval(rearchivePollRef.current); rearchivePollRef.current = null }
|
||||||
|
setRearchiveState('idle')
|
||||||
|
setRearchiveError('')
|
||||||
if (!selectedEntry || !archiveId) {
|
if (!selectedEntry || !archiveId) {
|
||||||
setDetail(null)
|
setDetail(null)
|
||||||
setTags([])
|
setTags([])
|
||||||
|
|
@ -47,6 +53,12 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
}).catch(() => {})
|
}).catch(() => {})
|
||||||
}, [selectedEntry, archiveId])
|
}, [selectedEntry, archiveId])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
clearInterval(rearchivePollRef.current)
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
async function handleTitleSave() {
|
async function handleTitleSave() {
|
||||||
const newTitle = titleDraft.trim() || null
|
const newTitle = titleDraft.trim() || null
|
||||||
try {
|
try {
|
||||||
|
|
@ -97,6 +109,46 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function handleRearchive() {
|
||||||
|
if (!selectedEntry || !archiveId || rearchiveState === 'running') return
|
||||||
|
setRearchiveState('running')
|
||||||
|
setRearchiveError('')
|
||||||
|
try {
|
||||||
|
const { job_uid } = await rearchiveEntry(archiveId, selectedEntry.entry_uid)
|
||||||
|
// Poll for job completion at 500ms — same interval as CaptureDialog
|
||||||
|
rearchivePollRef.current = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const job = await pollCaptureJob(archiveId, job_uid)
|
||||||
|
if (job.status === 'completed') {
|
||||||
|
clearInterval(rearchivePollRef.current)
|
||||||
|
rearchivePollRef.current = null
|
||||||
|
setRearchiveState('done')
|
||||||
|
// Refresh entry detail so artifact list updates
|
||||||
|
const [det, tgs] = await Promise.all([
|
||||||
|
fetchEntryDetail(archiveId, selectedEntry.entry_uid),
|
||||||
|
fetchEntryTags(archiveId, selectedEntry.entry_uid),
|
||||||
|
])
|
||||||
|
setDetail(det)
|
||||||
|
setTags(tgs)
|
||||||
|
} else if (job.status === 'failed') {
|
||||||
|
clearInterval(rearchivePollRef.current)
|
||||||
|
rearchivePollRef.current = null
|
||||||
|
setRearchiveState('error')
|
||||||
|
setRearchiveError(job.error_text || 'Re-archive failed.')
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
clearInterval(rearchivePollRef.current)
|
||||||
|
rearchivePollRef.current = null
|
||||||
|
setRearchiveState('error')
|
||||||
|
setRearchiveError('Network error while polling.')
|
||||||
|
}
|
||||||
|
}, 500)
|
||||||
|
} catch (e) {
|
||||||
|
setRearchiveState('error')
|
||||||
|
setRearchiveError(e.message || 'Failed to start re-archive.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const metaRows = detail ? [
|
const metaRows = detail ? [
|
||||||
['Added', formatTimestamp(detail.summary.archived_at)],
|
['Added', formatTimestamp(detail.summary.archived_at)],
|
||||||
['Source', detail.summary.source_kind],
|
['Source', detail.summary.source_kind],
|
||||||
|
|
@ -244,6 +296,25 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{detail && (detail.summary.entity_kind === 'tweet' || detail.summary.entity_kind === 'tweet_thread') && (
|
||||||
|
<div className="rail-section">
|
||||||
|
<div className="rail-section-heading">Actions</div>
|
||||||
|
<button
|
||||||
|
className="rail-rearchive-btn"
|
||||||
|
onClick={handleRearchive}
|
||||||
|
disabled={rearchiveState === 'running'}
|
||||||
|
>
|
||||||
|
{rearchiveState === 'running' ? 'Re-archiving\u2026' : 'Re-archive'}
|
||||||
|
</button>
|
||||||
|
{rearchiveState === 'done' && (
|
||||||
|
<p className="form-msg form-msg--ok" style={{ marginTop: '6px' }}>Re-archived successfully.</p>
|
||||||
|
)}
|
||||||
|
{rearchiveState === 'error' && (
|
||||||
|
<p className="form-msg form-msg--err" style={{ marginTop: '6px' }}>{rearchiveError}</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="rail-delete-zone">
|
<div className="rail-delete-zone">
|
||||||
<button className="rail-delete-btn" onClick={handleDeleteEntry}>
|
<button className="rail-delete-btn" onClick={handleDeleteEntry}>
|
||||||
Delete entry
|
Delete entry
|
||||||
|
|
|
||||||
|
|
@ -552,6 +552,24 @@ select {
|
||||||
border-color: var(--accent);
|
border-color: var(--accent);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.rail-rearchive-btn {
|
||||||
|
width: 100%;
|
||||||
|
padding: 6px 12px;
|
||||||
|
background: var(--surface-2, #2a2a2a);
|
||||||
|
color: var(--text, #e0e0e0);
|
||||||
|
border: 1px solid var(--border, #444);
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
.rail-rearchive-btn:hover:not(:disabled) {
|
||||||
|
background: var(--surface-3, #333);
|
||||||
|
}
|
||||||
|
.rail-rearchive-btn:disabled {
|
||||||
|
opacity: 0.6;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
/* ── Capture dialog ──────────────────────────────────────────────────────── */
|
/* ── Capture dialog ──────────────────────────────────────────────────────── */
|
||||||
.capture-dialog {
|
.capture-dialog {
|
||||||
border: 1px solid var(--line);
|
border: 1px solid var(--line);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue