mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
Compare commits
2 commits
2e8820a0da
...
2779afee2d
| Author | SHA1 | Date | |
|---|---|---|---|
| 2779afee2d | |||
| 03390362c5 |
16 changed files with 675 additions and 215 deletions
|
|
@ -742,6 +742,52 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
|
|||
}
|
||||
}
|
||||
|
||||
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
|
||||
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
|
||||
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
|
||||
fn register_tweet_artifacts(
|
||||
conn: &rusqlite::Connection,
|
||||
store_path: &Path,
|
||||
entry_id: i64,
|
||||
tweet_json_relpaths: &[String],
|
||||
) -> Result<()> {
|
||||
for relpath in tweet_json_relpaths {
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id,
|
||||
artifact_role: "raw_tweet_json".to_string(),
|
||||
storage_area: "raw_tweets".to_string(),
|
||||
relpath: relpath.clone(),
|
||||
blob_id: None,
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
let json_path = store_path.join(relpath);
|
||||
let json_str = fs::read_to_string(&json_path)
|
||||
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
|
||||
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
|
||||
let raw_path = PathBuf::from(&raw_relpath);
|
||||
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
||||
let blob_id = database::upsert_blob(conn, &blob)?;
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id,
|
||||
artifact_role: role,
|
||||
storage_area: "raw".to_string(),
|
||||
relpath: raw_relpath,
|
||||
blob_id: Some(blob_id),
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_tweet_entry(
|
||||
conn: &rusqlite::Connection,
|
||||
store_path: &Path,
|
||||
|
|
@ -751,6 +797,7 @@ fn record_tweet_entry(
|
|||
requested_locator: &str,
|
||||
source: Source,
|
||||
tweet_id: &str,
|
||||
tweet_json_relpaths: &[String],
|
||||
) -> Result<database::ArchivedEntry> {
|
||||
debug_assert!(run.run_uid.starts_with("run_"));
|
||||
debug_assert!(item.item_uid.starts_with("item_"));
|
||||
|
|
@ -764,7 +811,7 @@ fn record_tweet_entry(
|
|||
Some(&canonical_locator),
|
||||
&canonical_locator,
|
||||
)?;
|
||||
// Read tweet JSON early to extract title before entry creation
|
||||
// Read the primary tweet JSON to extract title before entry creation.
|
||||
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
|
||||
let tweet_meta = tweet_metadata_from_json(&tweet_json);
|
||||
|
|
@ -794,36 +841,8 @@ fn record_tweet_entry(
|
|||
)?;
|
||||
create_structured_root(store_path, &entry)?;
|
||||
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id: entry.id,
|
||||
artifact_role: "raw_tweet_json".to_string(),
|
||||
storage_area: "raw_tweets".to_string(),
|
||||
relpath: path_to_store_string(&tweet_json_relpath),
|
||||
blob_id: None,
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
|
||||
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
|
||||
let raw_path = PathBuf::from(&raw_relpath);
|
||||
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
||||
let blob_id = database::upsert_blob(conn, &blob)?;
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id: entry.id,
|
||||
artifact_role: role,
|
||||
storage_area: "raw".to_string(),
|
||||
relpath: raw_relpath,
|
||||
blob_id: Some(blob_id),
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
// Register all tweet JSONs and their blobs.
|
||||
register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?;
|
||||
|
||||
database::complete_archive_run_item(conn, item.id, entry.id)?;
|
||||
Ok(entry)
|
||||
|
|
@ -900,14 +919,12 @@ pub fn perform_capture(
|
|||
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
|
||||
Ok(downloader::http::UrlKind::File) => {}
|
||||
Err(e) => {
|
||||
// Record a failed item using the pre-probe source_kind so
|
||||
// failed_count increments and the item carries error_text.
|
||||
let (probe_sk, probe_ek, _) = source_metadata(Source::Url);
|
||||
let item = database::create_archive_run_item(
|
||||
&conn, run.id, None, 0, locator, None, probe_sk, probe_ek,
|
||||
)?;
|
||||
let msg = format!("Failed to probe URL: {e}");
|
||||
return Err(fail_run(&conn, &run, &item, &msg));
|
||||
// Probe failed (e.g. Cloudflare JS challenge, 403, network
|
||||
// issue). Fall back to treating the URL as an HTML page so
|
||||
// that the SingleFile/Chromium path can try — a real browser
|
||||
// can pass bot challenges that a plain HTTP client cannot.
|
||||
eprintln!("warn: probe failed for {locator}, assuming HTML: {e}");
|
||||
source = Source::WebPage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1186,7 +1203,7 @@ pub fn perform_capture(
|
|||
×tamp,
|
||||
&tweet_cookies,
|
||||
) {
|
||||
Ok(_) => {
|
||||
Ok(tweet_json_relpaths) => {
|
||||
let tweet_entry = record_tweet_entry(
|
||||
&conn,
|
||||
store_path,
|
||||
|
|
@ -1196,6 +1213,7 @@ pub fn perform_capture(
|
|||
locator,
|
||||
source,
|
||||
&tweet_id,
|
||||
&tweet_json_relpaths,
|
||||
)?;
|
||||
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
|
||||
database::finish_archive_run(&conn, run.id)?;
|
||||
|
|
@ -1364,6 +1382,107 @@ pub fn perform_capture(
|
|||
})
|
||||
}
|
||||
|
||||
/// Result of a tweet re-archive operation.
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct RearchiveResult {
|
||||
/// One of: "completed", "not_a_tweet", "scraper_failed".
|
||||
pub status: String,
|
||||
/// Human-readable detail (empty string for "completed").
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Re-archives an existing tweet or tweet_thread entry in-place.
|
||||
///
|
||||
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
|
||||
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
|
||||
/// returns `scraper_failed` WITHOUT touching the existing data.
|
||||
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
|
||||
/// and their media blobs. Does not change archived_at, title, tags, or collections.
|
||||
pub fn perform_rearchive(
|
||||
archive_paths: &ArchivePaths,
|
||||
entry_uid: &str,
|
||||
config: &CaptureConfig,
|
||||
) -> Result<RearchiveResult> {
|
||||
let store_path = &archive_paths.store_path;
|
||||
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
|
||||
|
||||
// Look up the entry.
|
||||
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
|
||||
None => {
|
||||
return Ok(RearchiveResult {
|
||||
status: "not_a_tweet".to_string(),
|
||||
message: "entry not found".to_string(),
|
||||
});
|
||||
}
|
||||
Some(e) => e,
|
||||
};
|
||||
|
||||
// Only tweet / tweet_thread entries can be re-archived this way.
|
||||
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
|
||||
return Ok(RearchiveResult {
|
||||
status: "not_a_tweet".to_string(),
|
||||
message: format!("entry is '{}', not a tweet", entry.entity_kind),
|
||||
});
|
||||
}
|
||||
|
||||
// Parse source_metadata_json for tweet_id and requested_locator.
|
||||
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
|
||||
.unwrap_or(serde_json::Value::Object(Default::default()));
|
||||
let requested_locator = meta["requested_locator"]
|
||||
.as_str()
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
|
||||
if requested_locator.is_empty() || tweet_id.is_empty() {
|
||||
return Ok(RearchiveResult {
|
||||
status: "not_a_tweet".to_string(),
|
||||
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let is_thread = entry.entity_kind == "tweet_thread";
|
||||
let timestamp = format!(
|
||||
"{}-{}",
|
||||
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
);
|
||||
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
|
||||
|
||||
// Run the scraper (staged). If this fails, existing data is untouched.
|
||||
let tweet_json_relpaths = match downloader::tweets::rearchive(
|
||||
&requested_locator,
|
||||
is_thread,
|
||||
store_path,
|
||||
×tamp,
|
||||
&tweet_cookies,
|
||||
) {
|
||||
Ok(relpaths) => relpaths,
|
||||
Err(e) => {
|
||||
return Ok(RearchiveResult {
|
||||
status: "scraper_failed".to_string(),
|
||||
message: format!("{e:#}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Atomically swap artifact rows: delete old, insert new.
|
||||
{
|
||||
let tx = conn.transaction()?;
|
||||
database::delete_entry_artifacts(&tx, entry.id)?;
|
||||
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
|
||||
tx.commit()?;
|
||||
}
|
||||
|
||||
database::refresh_entry_cached_bytes(&conn, entry.id)?;
|
||||
|
||||
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
|
||||
|
||||
Ok(RearchiveResult {
|
||||
status: "completed".to_string(),
|
||||
message: String::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn mime_for_font_ext(ext: &str) -> Option<String> {
|
||||
match ext {
|
||||
".woff2" => Some("font/woff2".to_string()),
|
||||
|
|
@ -1990,6 +2109,7 @@ mod tests {
|
|||
"tweet:123",
|
||||
Source::Tweet,
|
||||
"123",
|
||||
&["raw_tweets/tweet-123.json".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
database::finish_archive_run(&conn, run.id).unwrap();
|
||||
|
|
|
|||
|
|
@ -2391,6 +2391,43 @@ fn humanize_slug(slug: &str) -> String {
|
|||
.join(" ")
|
||||
}
|
||||
|
||||
/// A minimal view of an archived entry needed for re-archiving.
|
||||
pub struct EntryForRearchive {
|
||||
pub id: i64,
|
||||
pub entity_kind: String,
|
||||
pub source_metadata_json: String,
|
||||
}
|
||||
|
||||
/// Looks up an entry by its public UID for re-archive purposes.
|
||||
pub fn get_entry_for_rearchive(
|
||||
conn: &Connection,
|
||||
entry_uid: &str,
|
||||
) -> Result<Option<EntryForRearchive>> {
|
||||
conn.query_row(
|
||||
"SELECT id, entity_kind, source_metadata_json \
|
||||
FROM archived_entries WHERE entry_uid = ?1",
|
||||
[entry_uid],
|
||||
|row| {
|
||||
Ok(EntryForRearchive {
|
||||
id: row.get(0)?,
|
||||
entity_kind: row.get(1)?,
|
||||
source_metadata_json: row.get(2)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
|
||||
/// Deletes all `entry_artifacts` rows for the given entry.
|
||||
/// Call within a transaction for atomicity with re-insertion.
|
||||
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
|
||||
Ok(conn.execute(
|
||||
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
|
||||
[entry_id],
|
||||
)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pub enum UrlKind {
|
|||
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::limited(10))
|
||||
.user_agent("archivr/0.1")
|
||||
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
|
||||
.build()
|
||||
.context("failed to build HTTP client")?;
|
||||
|
||||
|
|
@ -86,7 +86,7 @@ pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<Ur
|
|||
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::limited(10))
|
||||
.user_agent("archivr/0.1")
|
||||
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
|
||||
.build()
|
||||
.context("failed to build HTTP client")?;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use regex::Regex;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
collections::HashMap,
|
||||
env,
|
||||
ffi::OsString,
|
||||
fs,
|
||||
|
|
@ -59,63 +59,39 @@ fn build_scraper_args(
|
|||
args
|
||||
}
|
||||
|
||||
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
|
||||
///
|
||||
/// Invokes the Python scraper, then moves all produced media assets into the
|
||||
/// content-addressed raw store and rewrites the JSON output to use the new
|
||||
/// store-relative paths. Returns `true` if new content was archived, `false`
|
||||
/// if the tweet was already present and `thread` is `false`.
|
||||
///
|
||||
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
||||
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
||||
pub fn archive(
|
||||
path: &str,
|
||||
/// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs
|
||||
/// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`).
|
||||
/// The staging dir starts empty so all files found there are exactly the touched set.
|
||||
fn run_scraper_staged(
|
||||
tweet_id: &str,
|
||||
thread: bool,
|
||||
store_path: &Path,
|
||||
timestamp: &str,
|
||||
cookies: &HashMap<String, String>,
|
||||
) -> Result<bool> {
|
||||
) -> Result<Vec<String>> {
|
||||
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
||||
// Output directory for Tweet JSON files.
|
||||
|
||||
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
|
||||
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
|
||||
// Scraper media-dir = staging_dir/media/
|
||||
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
|
||||
let staging_tweets_dir = staging_dir.join("raw_tweets");
|
||||
fs::create_dir_all(&staging_tweets_dir)?;
|
||||
|
||||
// Final destination for tweet JSONs.
|
||||
let output_dir = store_path.join("raw_tweets");
|
||||
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
|
||||
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
|
||||
fs::create_dir_all(&output_dir)?;
|
||||
fs::create_dir_all(&temp_dir)?;
|
||||
|
||||
// Path to the root - the to-be-archived tweet's JSON file.
|
||||
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
|
||||
if !thread && root_json.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let before = tweet_json_files(&output_dir)?;
|
||||
|
||||
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
||||
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
||||
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
||||
|
||||
// Credentials: only use cookie rules as Twitter credentials when the resolved
|
||||
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
|
||||
// requires. A global cookie rule for an unrelated site must not suppress the
|
||||
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
|
||||
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
|
||||
// The temp file is written in the semicolon-delimited format the Python scraper
|
||||
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
|
||||
// subprocess returns so secrets are never left on disk on failure.
|
||||
let temp_creds_path: Option<PathBuf>;
|
||||
let credentials_file: PathBuf;
|
||||
|
||||
let has_twitter_cookies =
|
||||
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||
|
||||
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||
if has_twitter_cookies {
|
||||
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
||||
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
|
||||
let creds_str = cookies
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}={v}"))
|
||||
|
|
@ -135,7 +111,7 @@ pub fn archive(
|
|||
let mut f = std::fs::File::create(&cf)
|
||||
.context("failed to write twitter credentials file")?;
|
||||
f.write_all(creds_str.as_bytes())
|
||||
.context("failed to write twitter credentials file")?;
|
||||
.context("failed to write twitter credentials file")?
|
||||
}
|
||||
temp_creds_path = Some(cf.clone());
|
||||
credentials_file = cf;
|
||||
|
|
@ -156,28 +132,21 @@ pub fn archive(
|
|||
}
|
||||
|
||||
let mut cmd = Command::new(&python);
|
||||
cmd.current_dir(&temp_dir).arg(&scraper_path);
|
||||
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
|
||||
// Run the scraper from staging_dir so relative paths in JSON resolve correctly.
|
||||
cmd.current_dir(&staging_dir).arg(&scraper_path);
|
||||
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
|
||||
// Hold the Result so we can delete the credentials file before propagating
|
||||
// any error — including spawn failures.
|
||||
let spawn_result = cmd.output().with_context(|| {
|
||||
format!(
|
||||
"Failed to spawn tweet scraper at {}",
|
||||
scraper_path.display()
|
||||
)
|
||||
format!("Failed to spawn tweet scraper at {}", scraper_path.display())
|
||||
});
|
||||
|
||||
// Remove temp credentials file unconditionally — secrets must not persist.
|
||||
if let Some(cf) = &temp_creds_path {
|
||||
let _ = fs::remove_file(cf);
|
||||
}
|
||||
|
||||
let output = spawn_result?;
|
||||
|
||||
if !output.status.success() {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
bail!(
|
||||
|
|
@ -187,7 +156,9 @@ pub fn archive(
|
|||
);
|
||||
}
|
||||
|
||||
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
|
||||
if !root_json.exists() {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
bail!(
|
||||
|
|
@ -198,13 +169,82 @@ pub fn archive(
|
|||
);
|
||||
}
|
||||
|
||||
cleanup_summary(&output_dir)?;
|
||||
let after = tweet_json_files(&output_dir)?;
|
||||
let new_jsons = new_tweet_jsons(&before, &after);
|
||||
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
|
||||
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
|
||||
// Remove the scraping_summary.json if the scraper left one.
|
||||
cleanup_summary(&staging_tweets_dir)?;
|
||||
|
||||
Ok(true)
|
||||
// Collect all tweet-*.json files from staging (this is the exact touched set).
|
||||
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
|
||||
})
|
||||
.collect();
|
||||
staged_jsons.sort();
|
||||
|
||||
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
|
||||
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
|
||||
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
|
||||
|
||||
// Move each staged JSON to its final destination, collecting store-relative relpaths.
|
||||
let mut relpaths = Vec::with_capacity(staged_jsons.len());
|
||||
for staged_path in &staged_jsons {
|
||||
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
|
||||
let dest = output_dir.join(filename);
|
||||
fs::rename(staged_path, &dest)
|
||||
.or_else(|_| {
|
||||
fs::copy(staged_path, &dest).map(|_| ())?;
|
||||
fs::remove_file(staged_path)
|
||||
})
|
||||
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
|
||||
let rel = dest
|
||||
.strip_prefix(store_path)
|
||||
.context("dest tweet JSON is not under store_path")?;
|
||||
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
|
||||
}
|
||||
|
||||
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
|
||||
Ok(relpaths)
|
||||
}
|
||||
|
||||
/// Archives a tweet (or full thread) identified by `path`.
|
||||
///
|
||||
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
|
||||
/// For single tweets already on disk, returns the existing file path without re-downloading.
|
||||
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
|
||||
pub fn archive(
|
||||
path: &str,
|
||||
thread: bool,
|
||||
store_path: &Path,
|
||||
timestamp: &str,
|
||||
cookies: &HashMap<String, String>,
|
||||
) -> Result<Vec<String>> {
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||
if !thread && root_json.exists() {
|
||||
// Already present; skip re-download, return the existing file for artifact registration.
|
||||
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
|
||||
}
|
||||
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
||||
}
|
||||
|
||||
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
|
||||
/// Always runs the scraper; output is staged before replacing existing files.
|
||||
/// Returns store-relative relpaths of all produced tweet JSON files.
|
||||
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
|
||||
pub fn rearchive(
|
||||
path: &str,
|
||||
thread: bool,
|
||||
store_path: &Path,
|
||||
timestamp: &str,
|
||||
cookies: &HashMap<String, String>,
|
||||
) -> Result<Vec<String>> {
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
||||
}
|
||||
|
||||
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
||||
|
|
@ -216,33 +256,6 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the set of `tweet-*.json` files present in `output_dir`.
|
||||
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
|
||||
let mut files = HashSet::new();
|
||||
|
||||
for entry in fs::read_dir(output_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.is_file()
|
||||
&& path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
|
||||
{
|
||||
files.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Returns the sorted list of JSON files present in `after` but not in `before`.
|
||||
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
|
||||
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
|
||||
files.sort();
|
||||
files
|
||||
}
|
||||
|
||||
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
||||
fn avatar_regex() -> &'static Regex {
|
||||
|
|
@ -519,9 +532,9 @@ mod tests {
|
|||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||
|
||||
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
|
||||
assert!(!archived);
|
||||
assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]);
|
||||
|
||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||
let _ = fs::remove_dir_all(store_path);
|
||||
|
|
@ -576,7 +589,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
|
|||
{
|
||||
"id": "$tweet_id",
|
||||
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
||||
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
|
||||
"author": { "avatar_local_path": "$media_dir/avatars/author.jpg" }
|
||||
}
|
||||
EOF
|
||||
"#,
|
||||
|
|
@ -592,16 +605,16 @@ EOF
|
|||
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
||||
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
||||
|
||||
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
let tweet_file = output_dir.join("tweet-123.json");
|
||||
let contents = fs::read_to_string(&tweet_file).unwrap();
|
||||
|
||||
assert!(archived);
|
||||
assert!(!relpaths.is_empty());
|
||||
assert!(tweet_file.exists());
|
||||
assert!(!output_dir.join("scraping_summary.json").exists());
|
||||
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
||||
assert!(contents.contains(r#""local_path": "raw/"#));
|
||||
assert!(!store_path.join("temp").join("ts").exists());
|
||||
assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists());
|
||||
|
||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
||||
|
|
|
|||
|
|
@ -221,6 +221,10 @@ pub fn app_with_state(state: AppState) -> Router {
|
|||
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
||||
get(serve_artifact),
|
||||
)
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
|
||||
post(rearchive_handler),
|
||||
)
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
||||
get(serve_entry_favicon),
|
||||
|
|
@ -832,6 +836,83 @@ async fn get_capture_job_handler(
|
|||
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
||||
}
|
||||
|
||||
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
|
||||
///
|
||||
/// Re-archives an existing tweet or tweet_thread entry in-place:
|
||||
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
|
||||
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
|
||||
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
|
||||
///
|
||||
/// Returns 202 immediately with a job_uid the client should poll via
|
||||
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
|
||||
async fn rearchive_handler(
|
||||
State(state): State<AppState>,
|
||||
auth_user: AuthUser,
|
||||
Path((archive_id, entry_uid)): Path<(String, String)>,
|
||||
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
|
||||
auth_user.require_role(ROLE_USER)?;
|
||||
let mounted = mounted_archive(&state, &archive_id)?;
|
||||
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// Create a capture job record so the client can poll for completion.
|
||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
||||
drop(conn);
|
||||
|
||||
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
|
||||
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
|
||||
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
||||
Err(_) => vec![],
|
||||
};
|
||||
let capture_config = capture::CaptureConfig {
|
||||
cookie_rules,
|
||||
ublock_enabled: None,
|
||||
cookie_ext_enabled: None,
|
||||
reader_mode: false,
|
||||
};
|
||||
|
||||
let job_uid_bg = job_uid.clone();
|
||||
let archive_path = mounted.archive_path.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let conn = match database::open_or_initialize(&archive_path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
|
||||
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
|
||||
Ok(result) => {
|
||||
if result.status == "completed" {
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "completed", None, None, None,
|
||||
).ok();
|
||||
} else {
|
||||
// "not_a_tweet" or "scraper_failed" — surface as a job failure
|
||||
// so the client sees a meaningful error message.
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "failed",
|
||||
None, Some(&result.message), None,
|
||||
).ok();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "failed",
|
||||
None, Some(&format!("{e:#}")), None,
|
||||
).ok();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok((
|
||||
StatusCode::ACCEPTED,
|
||||
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
|
||||
))
|
||||
}
|
||||
|
||||
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
||||
///
|
||||
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
45
crates/archivr-server/static/assets/index-DiHAMIVH.js
Normal file
45
crates/archivr-server/static/assets/index-DiHAMIVH.js
Normal file
File diff suppressed because one or more lines are too long
1
crates/archivr-server/static/assets/index-DwI288Sc.css
Normal file
1
crates/archivr-server/static/assets/index-DwI288Sc.css
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@
|
|||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Archivr</title>
|
||||
<script type="module" crossorigin src="/assets/index-D6iX-c2T.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-C0BIEdow.css">
|
||||
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
|
|
|||
|
|
@ -241,10 +241,12 @@ export default function App() {
|
|||
])
|
||||
}, [archiveId, searchQuery, tagFilter, loadEntries])
|
||||
|
||||
const handleToast = useCallback((text, locator, type = 'error') => {
|
||||
if (type === 'warning' && ublockWarningIgnored) return
|
||||
const handleToast = useCallback((text, locator, type = 'error', headline = null) => {
|
||||
// Only suppress per-item ublock/cookie warnings (those carry a locator).
|
||||
// Batch summary warnings (locator = null) must always show.
|
||||
if (type === 'warning' && ublockWarningIgnored && locator) return
|
||||
const id = ++toastIdRef.current
|
||||
setToasts(prev => [...prev, { id, text, locator, type }])
|
||||
setToasts(prev => [...prev, { id, text, locator, type, headline }])
|
||||
}, [ublockWarningIgnored])
|
||||
|
||||
const handleDismissToast = useCallback((id) => {
|
||||
|
|
@ -254,7 +256,7 @@ export default function App() {
|
|||
const handleIgnoreUblock = useCallback(() => {
|
||||
sessionStorage.setItem('ublockWarningIgnored', 'true')
|
||||
setUblockWarningIgnored(true)
|
||||
setToasts(prev => prev.filter(t => t.type !== 'warning'))
|
||||
setToasts(prev => prev.filter(t => !(t.type === 'warning' && t.locator)))
|
||||
}, [])
|
||||
|
||||
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;
|
||||
|
|
|
|||
|
|
@ -371,6 +371,18 @@ export async function deleteOrphanBlobs(archiveId) {
|
|||
return res.json()
|
||||
}
|
||||
|
||||
// ── Re-archive ────────────────────────────────────────────────────────────────
|
||||
export async function rearchiveEntry(archiveId, entryUid) {
|
||||
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}/rearchive`, {
|
||||
method: 'POST',
|
||||
})
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}))
|
||||
throw new Error(body.message || `rearchive failed: ${res.status}`)
|
||||
}
|
||||
return res.json() // { job_uid, status: 'pending' }
|
||||
}
|
||||
|
||||
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
||||
|
||||
export async function listCookieRules() {
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
const pollIntervals = useRef(new Map())
|
||||
// itemId → debounce timeoutId for probe calls
|
||||
const probeTimers = useRef(new Map())
|
||||
// batchId → { total, archived, warnings, failed }; only populated for multi-URL submits
|
||||
const batchRef = useRef(new Map())
|
||||
// stable ref so probe callbacks always see the current archiveId
|
||||
const archiveIdRef = useRef(archiveId)
|
||||
useEffect(() => { archiveIdRef.current = archiveId }, [archiveId])
|
||||
|
|
@ -194,7 +196,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
}
|
||||
}, [])
|
||||
|
||||
function startPolling(itemId, jobUid, locator, aid) {
|
||||
function startPolling(itemId, jobUid, locator, aid, batchId = null) {
|
||||
if (pollIntervals.current.has(jobUid)) return // already polling
|
||||
const intervalId = setInterval(async () => {
|
||||
try {
|
||||
|
|
@ -202,7 +204,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
if (updated.status === 'completed') {
|
||||
clearInterval(pollIntervals.current.get(jobUid))
|
||||
pollIntervals.current.delete(jobUid)
|
||||
// Show ✓ briefly then remove the row; if last row add a fresh one
|
||||
setItems(prev => prev.map(it => it.id === itemId ? { ...it, status: 'completed' } : it))
|
||||
setTimeout(() => {
|
||||
setItems(prev => {
|
||||
|
|
@ -211,13 +212,25 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
})
|
||||
}, 1400)
|
||||
onCapturedRef.current()
|
||||
// Warn if uBlock was requested but the extension wasn't available
|
||||
try {
|
||||
const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null
|
||||
if (notes?.ublock_skipped || notes?.cookie_ext_skipped) {
|
||||
onToastRef.current(null, locator, 'warning')
|
||||
const both = notes.ublock_skipped && notes.cookie_ext_skipped
|
||||
const msg = both
|
||||
? 'Captured without ad-blocking or cookie-consent extension. Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'
|
||||
: notes.ublock_skipped
|
||||
? 'Captured without ad-blocking. ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid.'
|
||||
: 'Captured without cookie-consent extension. ARCHIVR_COOKIE_EXT is not set or the path is invalid.'
|
||||
onToastRef.current(msg, locator, 'warning')
|
||||
settleBatch(batchId, 'warning', locator)
|
||||
} else {
|
||||
if (!batchId) onToastRef.current(null, locator, 'success')
|
||||
settleBatch(batchId, 'archived')
|
||||
}
|
||||
} catch {}
|
||||
} catch {
|
||||
if (!batchId) onToastRef.current(null, locator, 'success')
|
||||
settleBatch(batchId, 'archived')
|
||||
}
|
||||
} else if (updated.status === 'failed') {
|
||||
clearInterval(pollIntervals.current.get(jobUid))
|
||||
pollIntervals.current.delete(jobUid)
|
||||
|
|
@ -226,6 +239,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
it.id === itemId ? { ...it, status: 'failed', error: errText } : it
|
||||
))
|
||||
onToastRef.current(errText, locator)
|
||||
settleBatch(batchId, 'failed', locator)
|
||||
}
|
||||
// 'pending' / 'running': keep polling
|
||||
} catch (e) {
|
||||
|
|
@ -236,14 +250,51 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
it.id === itemId ? { ...it, status: 'failed', error: msg } : it
|
||||
))
|
||||
onToastRef.current(msg, locator)
|
||||
settleBatch(batchId, 'failed', locator)
|
||||
}
|
||||
}, 500)
|
||||
pollIntervals.current.set(jobUid, intervalId)
|
||||
}
|
||||
|
||||
async function submitItem(item) {
|
||||
// Increments the batch counter for the given outcome and emits a summary
|
||||
// toast once all jobs in the batch have settled.
|
||||
// 'warning' counts as archived (succeeded with caveats); locator recorded for detail text.
|
||||
function settleBatch(batchId, outcome, locator = null) {
|
||||
if (!batchId) return
|
||||
const batch = batchRef.current.get(batchId)
|
||||
if (!batch) return
|
||||
if (outcome === 'archived' || outcome === 'warning') {
|
||||
batch.archived++
|
||||
if (outcome === 'warning') { batch.warnings++; if (locator) batch.warningLocators.push(locator) }
|
||||
} else {
|
||||
batch.failed++
|
||||
if (locator) batch.failedLocators.push(locator)
|
||||
}
|
||||
if (batch.archived + batch.failed < batch.total) return
|
||||
// All settled — build headline + detail text, then emit summary and clean up.
|
||||
batchRef.current.delete(batchId)
|
||||
const { archived, warnings, failed, failedLocators, warningLocators } = batch
|
||||
let headline
|
||||
if (archived === 0) {
|
||||
headline = `${failed} failed`
|
||||
} else {
|
||||
const archivedStr = warnings > 0
|
||||
? `${archived} archived (${warnings} with warnings)`
|
||||
: `${archived} archived`
|
||||
headline = failed > 0 ? `${archivedStr}, ${failed} failed` : archivedStr
|
||||
}
|
||||
const type = archived === 0 ? 'error' : (failed > 0 || warnings > 0) ? 'warning' : 'success'
|
||||
const parts = []
|
||||
if (failedLocators.length > 0) parts.push(`Failed:\n${failedLocators.map(l => ` ${l}`).join('\n')}`)
|
||||
if (warningLocators.length > 0) parts.push(`With warnings:\n${warningLocators.map(l => ` ${l}`).join('\n')}`)
|
||||
const text = parts.length > 0 ? parts.join('\n') : null
|
||||
onToastRef.current(text, null, type, headline)
|
||||
}
|
||||
|
||||
|
||||
async function submitItem(item, batchId = null) {
|
||||
if (!item.locator.trim()) return
|
||||
const aid = archiveId // capture at submit time
|
||||
const aid = archiveId
|
||||
const loc = item.locator.trim()
|
||||
const qual = item.quality || 'best'
|
||||
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
|
||||
|
|
@ -253,17 +304,27 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
setItems(prev => prev.map(it =>
|
||||
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
|
||||
))
|
||||
startPolling(item.id, job.job_uid, loc, aid)
|
||||
startPolling(item.id, job.job_uid, loc, aid, batchId)
|
||||
} catch (e) {
|
||||
const msg = e.message || 'Submission failed.'
|
||||
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'failed', error: msg } : it))
|
||||
onToastRef.current(msg, loc)
|
||||
settleBatch(batchId, 'failed', loc)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function handleArchive() {
|
||||
const toSubmit = items.filter(it => it.status === 'idle' && it.locator.trim())
|
||||
toSubmit.forEach(it => submitItem(it))
|
||||
if (toSubmit.length === 0) return
|
||||
const batchId = toSubmit.length > 1
|
||||
? (crypto.randomUUID?.() ?? `batch-${Date.now()}`)
|
||||
: null
|
||||
if (batchId) {
|
||||
batchRef.current.set(batchId, { total: toSubmit.length, archived: 0, warnings: 0, failed: 0, failedLocators: [], warningLocators: [] })
|
||||
}
|
||||
toSubmit.forEach(it => submitItem(it, batchId))
|
||||
dialogRef.current?.close()
|
||||
}
|
||||
|
||||
function addRow() {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useState, useEffect, useRef } from 'react'
|
||||
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry } from '../api'
|
||||
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry, rearchiveEntry, pollCaptureJob } from '../api'
|
||||
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
|
||||
|
||||
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
|
||||
|
|
@ -21,8 +21,14 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
const titleCancelRef = useRef(false)
|
||||
const [editingTitle, setEditingTitle] = useState(false)
|
||||
const [titleDraft, setTitleDraft] = useState('')
|
||||
const [rearchiveState, setRearchiveState] = useState('idle') // 'idle' | 'running' | 'done' | 'error'
|
||||
const [rearchiveError, setRearchiveError] = useState('')
|
||||
const rearchivePollRef = useRef(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (rearchivePollRef.current) { clearInterval(rearchivePollRef.current); rearchivePollRef.current = null }
|
||||
setRearchiveState('idle')
|
||||
setRearchiveError('')
|
||||
if (!selectedEntry || !archiveId) {
|
||||
setDetail(null)
|
||||
setTags([])
|
||||
|
|
@ -47,6 +53,12 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
}).catch(() => {})
|
||||
}, [selectedEntry, archiveId])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
}
|
||||
}, [])
|
||||
|
||||
async function handleTitleSave() {
|
||||
const newTitle = titleDraft.trim() || null
|
||||
try {
|
||||
|
|
@ -97,6 +109,46 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
}
|
||||
}
|
||||
|
||||
async function handleRearchive() {
|
||||
if (!selectedEntry || !archiveId || rearchiveState === 'running') return
|
||||
setRearchiveState('running')
|
||||
setRearchiveError('')
|
||||
try {
|
||||
const { job_uid } = await rearchiveEntry(archiveId, selectedEntry.entry_uid)
|
||||
// Poll for job completion at 500ms — same interval as CaptureDialog
|
||||
rearchivePollRef.current = setInterval(async () => {
|
||||
try {
|
||||
const job = await pollCaptureJob(archiveId, job_uid)
|
||||
if (job.status === 'completed') {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
rearchivePollRef.current = null
|
||||
setRearchiveState('done')
|
||||
// Refresh entry detail so artifact list updates
|
||||
const [det, tgs] = await Promise.all([
|
||||
fetchEntryDetail(archiveId, selectedEntry.entry_uid),
|
||||
fetchEntryTags(archiveId, selectedEntry.entry_uid),
|
||||
])
|
||||
setDetail(det)
|
||||
setTags(tgs)
|
||||
} else if (job.status === 'failed') {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
rearchivePollRef.current = null
|
||||
setRearchiveState('error')
|
||||
setRearchiveError(job.error_text || 'Re-archive failed.')
|
||||
}
|
||||
} catch {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
rearchivePollRef.current = null
|
||||
setRearchiveState('error')
|
||||
setRearchiveError('Network error while polling.')
|
||||
}
|
||||
}, 500)
|
||||
} catch (e) {
|
||||
setRearchiveState('error')
|
||||
setRearchiveError(e.message || 'Failed to start re-archive.')
|
||||
}
|
||||
}
|
||||
|
||||
const metaRows = detail ? [
|
||||
['Added', formatTimestamp(detail.summary.archived_at)],
|
||||
['Source', detail.summary.source_kind],
|
||||
|
|
@ -244,6 +296,25 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
</div>
|
||||
)}
|
||||
|
||||
{detail && (detail.summary.entity_kind === 'tweet' || detail.summary.entity_kind === 'tweet_thread') && (
|
||||
<div className="rail-section">
|
||||
<div className="rail-section-heading">Actions</div>
|
||||
<button
|
||||
className="rail-rearchive-btn"
|
||||
onClick={handleRearchive}
|
||||
disabled={rearchiveState === 'running'}
|
||||
>
|
||||
{rearchiveState === 'running' ? 'Re-archiving\u2026' : 'Re-archive'}
|
||||
</button>
|
||||
{rearchiveState === 'done' && (
|
||||
<p className="form-msg form-msg--ok" style={{ marginTop: '6px' }}>Re-archived successfully.</p>
|
||||
)}
|
||||
{rearchiveState === 'error' && (
|
||||
<p className="form-msg form-msg--err" style={{ marginTop: '6px' }}>{rearchiveError}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="rail-delete-zone">
|
||||
<button className="rail-delete-btn" onClick={handleDeleteEntry}>
|
||||
Delete entry
|
||||
|
|
|
|||
|
|
@ -13,9 +13,27 @@ export default function ToastStack({ toasts, onDismiss, onIgnoreUblock }) {
|
|||
)
|
||||
}
|
||||
|
||||
// For URLs: show hostname/…/tail (drops protocol + middle path, keeps domain + identifier).
|
||||
// For shorthands (yt:, x:, etc.): keep the tail since it's the whole identifier.
|
||||
// CSS text-overflow:ellipsis on .toast-locator handles any remaining overflow.
|
||||
function shortLocator(locator) {
|
||||
if (!locator) return null
|
||||
if (locator.length <= 52) return locator
|
||||
try {
|
||||
const { hostname, pathname, search } = new URL(locator)
|
||||
const segments = pathname.split('/').filter(Boolean)
|
||||
const tail = (segments[segments.length - 1] ?? '') + search
|
||||
const candidate = tail ? `${hostname}/\u2026/${tail}` : hostname
|
||||
return candidate.length <= 56 ? candidate : candidate.slice(0, 53) + '\u2026'
|
||||
} catch {
|
||||
return '\u2026' + locator.slice(-51)
|
||||
}
|
||||
}
|
||||
|
||||
function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||
const [expanded, setExpanded] = useState(false)
|
||||
const isWarning = toast.type === 'warning'
|
||||
const isSuccess = toast.type === 'success'
|
||||
|
||||
// Auto-dismiss after TTL; paused while detail is expanded
|
||||
useEffect(() => {
|
||||
|
|
@ -24,9 +42,24 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
return () => clearTimeout(timer)
|
||||
}, [expanded, toast.id, onDismiss])
|
||||
|
||||
const short = toast.locator
|
||||
? (toast.locator.length > 48 ? toast.locator.slice(0, 45) + '\u2026' : toast.locator)
|
||||
: null
|
||||
const short = shortLocator(toast.locator)
|
||||
|
||||
if (isSuccess) {
|
||||
return (
|
||||
<div className="toast toast--success" role="alert" aria-atomic="true">
|
||||
<div className="toast-top">
|
||||
<span className="toast-icon" aria-hidden="true">✓</span>
|
||||
<div className="toast-body">
|
||||
<span className="toast-headline">{toast.headline || 'Archived'}</span>
|
||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
||||
</div>
|
||||
<div className="toast-btns">
|
||||
<button type="button" className="toast-dismiss" onClick={() => onDismiss(toast.id)} aria-label="Dismiss">×</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isWarning) {
|
||||
return (
|
||||
|
|
@ -34,25 +67,29 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
<div className="toast-top">
|
||||
<span className="toast-icon" aria-hidden="true">⚠</span>
|
||||
<div className="toast-body">
|
||||
<span className="toast-headline">Ad-blocking unavailable</span>
|
||||
{short && <span className="toast-locator">{short}</span>}
|
||||
<span className="toast-headline">{toast.headline || 'Archived with warnings'}</span>
|
||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
||||
</div>
|
||||
<div className="toast-btns">
|
||||
<button
|
||||
type="button"
|
||||
className="toast-view-btn"
|
||||
onClick={() => setExpanded(v => !v)}
|
||||
aria-expanded={expanded}
|
||||
>
|
||||
{expanded ? 'Hide' : 'Details'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="toast-view-btn toast-ignore-btn"
|
||||
onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }}
|
||||
>
|
||||
Ignore
|
||||
</button>
|
||||
{toast.text && (
|
||||
<button
|
||||
type="button"
|
||||
className="toast-view-btn"
|
||||
onClick={() => setExpanded(v => !v)}
|
||||
aria-expanded={expanded}
|
||||
>
|
||||
{expanded ? 'Hide' : 'Details'}
|
||||
</button>
|
||||
)}
|
||||
{toast.locator && (
|
||||
<button
|
||||
type="button"
|
||||
className="toast-view-btn toast-ignore-btn"
|
||||
onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }}
|
||||
>
|
||||
Ignore
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
className="toast-dismiss"
|
||||
|
|
@ -65,7 +102,7 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
</div>
|
||||
{expanded && (
|
||||
<p className="toast-warning-detail">
|
||||
{toast.text || 'ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid. The page was captured without ad-blocking. Set ARCHIVR_UBLOCK_EXT to the unpacked uBlock Origin Lite extension directory to enable ad-blocking, or set ARCHIVR_UBLOCK=false to silence this warning.'}
|
||||
{toast.text || 'The page was captured but one or more browser extensions were unavailable (ad-blocking or cookie-consent). Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -77,8 +114,8 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
<div className="toast-top">
|
||||
<span className="toast-icon" aria-hidden="true">✕</span>
|
||||
<div className="toast-body">
|
||||
<span className="toast-headline">Capture failed</span>
|
||||
{short && <span className="toast-locator">{short}</span>}
|
||||
<span className="toast-headline">{toast.headline || 'Capture failed'}</span>
|
||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
||||
</div>
|
||||
<div className="toast-btns">
|
||||
{toast.text && (
|
||||
|
|
|
|||
|
|
@ -552,6 +552,24 @@ select {
|
|||
border-color: var(--accent);
|
||||
}
|
||||
|
||||
.rail-rearchive-btn {
|
||||
width: 100%;
|
||||
padding: 6px 12px;
|
||||
background: var(--surface-2, #2a2a2a);
|
||||
color: var(--text, #e0e0e0);
|
||||
border: 1px solid var(--border, #444);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
}
|
||||
.rail-rearchive-btn:hover:not(:disabled) {
|
||||
background: var(--surface-3, #333);
|
||||
}
|
||||
.rail-rearchive-btn:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* ── Capture dialog ──────────────────────────────────────────────────────── */
|
||||
.capture-dialog {
|
||||
border: 1px solid var(--line);
|
||||
|
|
@ -869,6 +887,7 @@ select {
|
|||
}
|
||||
.toast--error { border-left: 3px solid var(--accent); }
|
||||
.toast--warning { border-left: 3px solid #e8a000; }
|
||||
.toast--success { border-left: 3px solid #22a855; }
|
||||
|
||||
.toast-top {
|
||||
display: flex;
|
||||
|
|
@ -883,6 +902,8 @@ select {
|
|||
color: var(--accent);
|
||||
margin-top: 2px;
|
||||
}
|
||||
.toast--success .toast-icon { color: #22a855; }
|
||||
.toast--warning .toast-icon { color: #e8a000; }
|
||||
.toast-body {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue