mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
Compare commits
No commits in common. "2779afee2dd36271c20a5384c98cfc71f0220885" and "2e8820a0dac027f70bcd364877b880111d0ab8ec" have entirely different histories.
2779afee2d
...
2e8820a0da
16 changed files with 215 additions and 675 deletions
|
|
@ -742,52 +742,6 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
|
|||
}
|
||||
}
|
||||
|
||||
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
|
||||
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
|
||||
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
|
||||
fn register_tweet_artifacts(
|
||||
conn: &rusqlite::Connection,
|
||||
store_path: &Path,
|
||||
entry_id: i64,
|
||||
tweet_json_relpaths: &[String],
|
||||
) -> Result<()> {
|
||||
for relpath in tweet_json_relpaths {
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id,
|
||||
artifact_role: "raw_tweet_json".to_string(),
|
||||
storage_area: "raw_tweets".to_string(),
|
||||
relpath: relpath.clone(),
|
||||
blob_id: None,
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
let json_path = store_path.join(relpath);
|
||||
let json_str = fs::read_to_string(&json_path)
|
||||
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
|
||||
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
|
||||
let raw_path = PathBuf::from(&raw_relpath);
|
||||
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
||||
let blob_id = database::upsert_blob(conn, &blob)?;
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id,
|
||||
artifact_role: role,
|
||||
storage_area: "raw".to_string(),
|
||||
relpath: raw_relpath,
|
||||
blob_id: Some(blob_id),
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_tweet_entry(
|
||||
conn: &rusqlite::Connection,
|
||||
store_path: &Path,
|
||||
|
|
@ -797,7 +751,6 @@ fn record_tweet_entry(
|
|||
requested_locator: &str,
|
||||
source: Source,
|
||||
tweet_id: &str,
|
||||
tweet_json_relpaths: &[String],
|
||||
) -> Result<database::ArchivedEntry> {
|
||||
debug_assert!(run.run_uid.starts_with("run_"));
|
||||
debug_assert!(item.item_uid.starts_with("item_"));
|
||||
|
|
@ -811,7 +764,7 @@ fn record_tweet_entry(
|
|||
Some(&canonical_locator),
|
||||
&canonical_locator,
|
||||
)?;
|
||||
// Read the primary tweet JSON to extract title before entry creation.
|
||||
// Read tweet JSON early to extract title before entry creation
|
||||
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
|
||||
let tweet_meta = tweet_metadata_from_json(&tweet_json);
|
||||
|
|
@ -841,8 +794,36 @@ fn record_tweet_entry(
|
|||
)?;
|
||||
create_structured_root(store_path, &entry)?;
|
||||
|
||||
// Register all tweet JSONs and their blobs.
|
||||
register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?;
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id: entry.id,
|
||||
artifact_role: "raw_tweet_json".to_string(),
|
||||
storage_area: "raw_tweets".to_string(),
|
||||
relpath: path_to_store_string(&tweet_json_relpath),
|
||||
blob_id: None,
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
|
||||
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
|
||||
let raw_path = PathBuf::from(&raw_relpath);
|
||||
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
||||
let blob_id = database::upsert_blob(conn, &blob)?;
|
||||
database::add_entry_artifact(
|
||||
conn,
|
||||
&database::NewArtifact {
|
||||
entry_id: entry.id,
|
||||
artifact_role: role,
|
||||
storage_area: "raw".to_string(),
|
||||
relpath: raw_relpath,
|
||||
blob_id: Some(blob_id),
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
database::complete_archive_run_item(conn, item.id, entry.id)?;
|
||||
Ok(entry)
|
||||
|
|
@ -919,12 +900,14 @@ pub fn perform_capture(
|
|||
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
|
||||
Ok(downloader::http::UrlKind::File) => {}
|
||||
Err(e) => {
|
||||
// Probe failed (e.g. Cloudflare JS challenge, 403, network
|
||||
// issue). Fall back to treating the URL as an HTML page so
|
||||
// that the SingleFile/Chromium path can try — a real browser
|
||||
// can pass bot challenges that a plain HTTP client cannot.
|
||||
eprintln!("warn: probe failed for {locator}, assuming HTML: {e}");
|
||||
source = Source::WebPage;
|
||||
// Record a failed item using the pre-probe source_kind so
|
||||
// failed_count increments and the item carries error_text.
|
||||
let (probe_sk, probe_ek, _) = source_metadata(Source::Url);
|
||||
let item = database::create_archive_run_item(
|
||||
&conn, run.id, None, 0, locator, None, probe_sk, probe_ek,
|
||||
)?;
|
||||
let msg = format!("Failed to probe URL: {e}");
|
||||
return Err(fail_run(&conn, &run, &item, &msg));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1203,7 +1186,7 @@ pub fn perform_capture(
|
|||
×tamp,
|
||||
&tweet_cookies,
|
||||
) {
|
||||
Ok(tweet_json_relpaths) => {
|
||||
Ok(_) => {
|
||||
let tweet_entry = record_tweet_entry(
|
||||
&conn,
|
||||
store_path,
|
||||
|
|
@ -1213,7 +1196,6 @@ pub fn perform_capture(
|
|||
locator,
|
||||
source,
|
||||
&tweet_id,
|
||||
&tweet_json_relpaths,
|
||||
)?;
|
||||
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
|
||||
database::finish_archive_run(&conn, run.id)?;
|
||||
|
|
@ -1382,107 +1364,6 @@ pub fn perform_capture(
|
|||
})
|
||||
}
|
||||
|
||||
/// Result of a tweet re-archive operation.
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct RearchiveResult {
|
||||
/// One of: "completed", "not_a_tweet", "scraper_failed".
|
||||
pub status: String,
|
||||
/// Human-readable detail (empty string for "completed").
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Re-archives an existing tweet or tweet_thread entry in-place.
|
||||
///
|
||||
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
|
||||
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
|
||||
/// returns `scraper_failed` WITHOUT touching the existing data.
|
||||
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
|
||||
/// and their media blobs. Does not change archived_at, title, tags, or collections.
|
||||
pub fn perform_rearchive(
|
||||
archive_paths: &ArchivePaths,
|
||||
entry_uid: &str,
|
||||
config: &CaptureConfig,
|
||||
) -> Result<RearchiveResult> {
|
||||
let store_path = &archive_paths.store_path;
|
||||
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
|
||||
|
||||
// Look up the entry.
|
||||
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
|
||||
None => {
|
||||
return Ok(RearchiveResult {
|
||||
status: "not_a_tweet".to_string(),
|
||||
message: "entry not found".to_string(),
|
||||
});
|
||||
}
|
||||
Some(e) => e,
|
||||
};
|
||||
|
||||
// Only tweet / tweet_thread entries can be re-archived this way.
|
||||
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
|
||||
return Ok(RearchiveResult {
|
||||
status: "not_a_tweet".to_string(),
|
||||
message: format!("entry is '{}', not a tweet", entry.entity_kind),
|
||||
});
|
||||
}
|
||||
|
||||
// Parse source_metadata_json for tweet_id and requested_locator.
|
||||
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
|
||||
.unwrap_or(serde_json::Value::Object(Default::default()));
|
||||
let requested_locator = meta["requested_locator"]
|
||||
.as_str()
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
|
||||
if requested_locator.is_empty() || tweet_id.is_empty() {
|
||||
return Ok(RearchiveResult {
|
||||
status: "not_a_tweet".to_string(),
|
||||
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let is_thread = entry.entity_kind == "tweet_thread";
|
||||
let timestamp = format!(
|
||||
"{}-{}",
|
||||
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
);
|
||||
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
|
||||
|
||||
// Run the scraper (staged). If this fails, existing data is untouched.
|
||||
let tweet_json_relpaths = match downloader::tweets::rearchive(
|
||||
&requested_locator,
|
||||
is_thread,
|
||||
store_path,
|
||||
×tamp,
|
||||
&tweet_cookies,
|
||||
) {
|
||||
Ok(relpaths) => relpaths,
|
||||
Err(e) => {
|
||||
return Ok(RearchiveResult {
|
||||
status: "scraper_failed".to_string(),
|
||||
message: format!("{e:#}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Atomically swap artifact rows: delete old, insert new.
|
||||
{
|
||||
let tx = conn.transaction()?;
|
||||
database::delete_entry_artifacts(&tx, entry.id)?;
|
||||
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
|
||||
tx.commit()?;
|
||||
}
|
||||
|
||||
database::refresh_entry_cached_bytes(&conn, entry.id)?;
|
||||
|
||||
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
|
||||
|
||||
Ok(RearchiveResult {
|
||||
status: "completed".to_string(),
|
||||
message: String::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn mime_for_font_ext(ext: &str) -> Option<String> {
|
||||
match ext {
|
||||
".woff2" => Some("font/woff2".to_string()),
|
||||
|
|
@ -2109,7 +1990,6 @@ mod tests {
|
|||
"tweet:123",
|
||||
Source::Tweet,
|
||||
"123",
|
||||
&["raw_tweets/tweet-123.json".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
database::finish_archive_run(&conn, run.id).unwrap();
|
||||
|
|
|
|||
|
|
@ -2391,43 +2391,6 @@ fn humanize_slug(slug: &str) -> String {
|
|||
.join(" ")
|
||||
}
|
||||
|
||||
/// A minimal view of an archived entry needed for re-archiving.
|
||||
pub struct EntryForRearchive {
|
||||
pub id: i64,
|
||||
pub entity_kind: String,
|
||||
pub source_metadata_json: String,
|
||||
}
|
||||
|
||||
/// Looks up an entry by its public UID for re-archive purposes.
|
||||
pub fn get_entry_for_rearchive(
|
||||
conn: &Connection,
|
||||
entry_uid: &str,
|
||||
) -> Result<Option<EntryForRearchive>> {
|
||||
conn.query_row(
|
||||
"SELECT id, entity_kind, source_metadata_json \
|
||||
FROM archived_entries WHERE entry_uid = ?1",
|
||||
[entry_uid],
|
||||
|row| {
|
||||
Ok(EntryForRearchive {
|
||||
id: row.get(0)?,
|
||||
entity_kind: row.get(1)?,
|
||||
source_metadata_json: row.get(2)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
|
||||
/// Deletes all `entry_artifacts` rows for the given entry.
|
||||
/// Call within a transaction for atomicity with re-insertion.
|
||||
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
|
||||
Ok(conn.execute(
|
||||
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
|
||||
[entry_id],
|
||||
)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pub enum UrlKind {
|
|||
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::limited(10))
|
||||
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
|
||||
.user_agent("archivr/0.1")
|
||||
.build()
|
||||
.context("failed to build HTTP client")?;
|
||||
|
||||
|
|
@ -86,7 +86,7 @@ pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<Ur
|
|||
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::limited(10))
|
||||
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
|
||||
.user_agent("archivr/0.1")
|
||||
.build()
|
||||
.context("failed to build HTTP client")?;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use regex::Regex;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
collections::{HashMap, HashSet},
|
||||
env,
|
||||
ffi::OsString,
|
||||
fs,
|
||||
|
|
@ -59,39 +59,63 @@ fn build_scraper_args(
|
|||
args
|
||||
}
|
||||
|
||||
/// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs
|
||||
/// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`).
|
||||
/// The staging dir starts empty so all files found there are exactly the touched set.
|
||||
fn run_scraper_staged(
|
||||
tweet_id: &str,
|
||||
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
|
||||
///
|
||||
/// Invokes the Python scraper, then moves all produced media assets into the
|
||||
/// content-addressed raw store and rewrites the JSON output to use the new
|
||||
/// store-relative paths. Returns `true` if new content was archived, `false`
|
||||
/// if the tweet was already present and `thread` is `false`.
|
||||
///
|
||||
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
||||
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
||||
pub fn archive(
|
||||
path: &str,
|
||||
thread: bool,
|
||||
store_path: &Path,
|
||||
timestamp: &str,
|
||||
cookies: &HashMap<String, String>,
|
||||
) -> Result<Vec<String>> {
|
||||
) -> Result<bool> {
|
||||
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
||||
|
||||
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
|
||||
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
|
||||
// Scraper media-dir = staging_dir/media/
|
||||
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
|
||||
let staging_tweets_dir = staging_dir.join("raw_tweets");
|
||||
fs::create_dir_all(&staging_tweets_dir)?;
|
||||
|
||||
// Final destination for tweet JSONs.
|
||||
// Output directory for Tweet JSON files.
|
||||
let output_dir = store_path.join("raw_tweets");
|
||||
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
|
||||
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
|
||||
fs::create_dir_all(&output_dir)?;
|
||||
fs::create_dir_all(&temp_dir)?;
|
||||
|
||||
// Path to the root - the to-be-archived tweet's JSON file.
|
||||
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
|
||||
if !thread && root_json.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let before = tweet_json_files(&output_dir)?;
|
||||
|
||||
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
||||
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
||||
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
||||
|
||||
// Credentials: only use cookie rules as Twitter credentials when the resolved
|
||||
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
|
||||
// requires. A global cookie rule for an unrelated site must not suppress the
|
||||
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
|
||||
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
|
||||
// The temp file is written in the semicolon-delimited format the Python scraper
|
||||
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
|
||||
// subprocess returns so secrets are never left on disk on failure.
|
||||
let temp_creds_path: Option<PathBuf>;
|
||||
let credentials_file: PathBuf;
|
||||
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||
|
||||
let has_twitter_cookies =
|
||||
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||
|
||||
if has_twitter_cookies {
|
||||
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
||||
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
|
||||
let creds_str = cookies
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}={v}"))
|
||||
|
|
@ -111,7 +135,7 @@ fn run_scraper_staged(
|
|||
let mut f = std::fs::File::create(&cf)
|
||||
.context("failed to write twitter credentials file")?;
|
||||
f.write_all(creds_str.as_bytes())
|
||||
.context("failed to write twitter credentials file")?
|
||||
.context("failed to write twitter credentials file")?;
|
||||
}
|
||||
temp_creds_path = Some(cf.clone());
|
||||
credentials_file = cf;
|
||||
|
|
@ -132,21 +156,28 @@ fn run_scraper_staged(
|
|||
}
|
||||
|
||||
let mut cmd = Command::new(&python);
|
||||
// Run the scraper from staging_dir so relative paths in JSON resolve correctly.
|
||||
cmd.current_dir(&staging_dir).arg(&scraper_path);
|
||||
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
|
||||
cmd.current_dir(&temp_dir).arg(&scraper_path);
|
||||
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
|
||||
// Hold the Result so we can delete the credentials file before propagating
|
||||
// any error — including spawn failures.
|
||||
let spawn_result = cmd.output().with_context(|| {
|
||||
format!("Failed to spawn tweet scraper at {}", scraper_path.display())
|
||||
format!(
|
||||
"Failed to spawn tweet scraper at {}",
|
||||
scraper_path.display()
|
||||
)
|
||||
});
|
||||
|
||||
// Remove temp credentials file unconditionally — secrets must not persist.
|
||||
if let Some(cf) = &temp_creds_path {
|
||||
let _ = fs::remove_file(cf);
|
||||
}
|
||||
|
||||
let output = spawn_result?;
|
||||
|
||||
if !output.status.success() {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
bail!(
|
||||
|
|
@ -156,9 +187,7 @@ fn run_scraper_staged(
|
|||
);
|
||||
}
|
||||
|
||||
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
|
||||
if !root_json.exists() {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
bail!(
|
||||
|
|
@ -169,82 +198,13 @@ fn run_scraper_staged(
|
|||
);
|
||||
}
|
||||
|
||||
// Remove the scraping_summary.json if the scraper left one.
|
||||
cleanup_summary(&staging_tweets_dir)?;
|
||||
cleanup_summary(&output_dir)?;
|
||||
let after = tweet_json_files(&output_dir)?;
|
||||
let new_jsons = new_tweet_jsons(&before, &after);
|
||||
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
|
||||
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
|
||||
|
||||
// Collect all tweet-*.json files from staging (this is the exact touched set).
|
||||
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
|
||||
})
|
||||
.collect();
|
||||
staged_jsons.sort();
|
||||
|
||||
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
|
||||
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
|
||||
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
|
||||
|
||||
// Move each staged JSON to its final destination, collecting store-relative relpaths.
|
||||
let mut relpaths = Vec::with_capacity(staged_jsons.len());
|
||||
for staged_path in &staged_jsons {
|
||||
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
|
||||
let dest = output_dir.join(filename);
|
||||
fs::rename(staged_path, &dest)
|
||||
.or_else(|_| {
|
||||
fs::copy(staged_path, &dest).map(|_| ())?;
|
||||
fs::remove_file(staged_path)
|
||||
})
|
||||
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
|
||||
let rel = dest
|
||||
.strip_prefix(store_path)
|
||||
.context("dest tweet JSON is not under store_path")?;
|
||||
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
|
||||
}
|
||||
|
||||
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
|
||||
Ok(relpaths)
|
||||
}
|
||||
|
||||
/// Archives a tweet (or full thread) identified by `path`.
|
||||
///
|
||||
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
|
||||
/// For single tweets already on disk, returns the existing file path without re-downloading.
|
||||
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
|
||||
pub fn archive(
|
||||
path: &str,
|
||||
thread: bool,
|
||||
store_path: &Path,
|
||||
timestamp: &str,
|
||||
cookies: &HashMap<String, String>,
|
||||
) -> Result<Vec<String>> {
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||
if !thread && root_json.exists() {
|
||||
// Already present; skip re-download, return the existing file for artifact registration.
|
||||
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
|
||||
}
|
||||
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
||||
}
|
||||
|
||||
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
|
||||
/// Always runs the scraper; output is staged before replacing existing files.
|
||||
/// Returns store-relative relpaths of all produced tweet JSON files.
|
||||
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
|
||||
pub fn rearchive(
|
||||
path: &str,
|
||||
thread: bool,
|
||||
store_path: &Path,
|
||||
timestamp: &str,
|
||||
cookies: &HashMap<String, String>,
|
||||
) -> Result<Vec<String>> {
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
||||
|
|
@ -256,6 +216,33 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the set of `tweet-*.json` files present in `output_dir`.
|
||||
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
|
||||
let mut files = HashSet::new();
|
||||
|
||||
for entry in fs::read_dir(output_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.is_file()
|
||||
&& path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
|
||||
{
|
||||
files.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Returns the sorted list of JSON files present in `after` but not in `before`.
|
||||
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
|
||||
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
|
||||
files.sort();
|
||||
files
|
||||
}
|
||||
|
||||
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
||||
fn avatar_regex() -> &'static Regex {
|
||||
|
|
@ -532,9 +519,9 @@ mod tests {
|
|||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||
|
||||
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
|
||||
assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]);
|
||||
assert!(!archived);
|
||||
|
||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||
let _ = fs::remove_dir_all(store_path);
|
||||
|
|
@ -589,7 +576,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
|
|||
{
|
||||
"id": "$tweet_id",
|
||||
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
||||
"author": { "avatar_local_path": "$media_dir/avatars/author.jpg" }
|
||||
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
|
||||
}
|
||||
EOF
|
||||
"#,
|
||||
|
|
@ -605,16 +592,16 @@ EOF
|
|||
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
||||
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
||||
|
||||
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||
let tweet_file = output_dir.join("tweet-123.json");
|
||||
let contents = fs::read_to_string(&tweet_file).unwrap();
|
||||
|
||||
assert!(!relpaths.is_empty());
|
||||
assert!(archived);
|
||||
assert!(tweet_file.exists());
|
||||
assert!(!output_dir.join("scraping_summary.json").exists());
|
||||
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
||||
assert!(contents.contains(r#""local_path": "raw/"#));
|
||||
assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists());
|
||||
assert!(!store_path.join("temp").join("ts").exists());
|
||||
|
||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
||||
|
|
|
|||
|
|
@ -221,10 +221,6 @@ pub fn app_with_state(state: AppState) -> Router {
|
|||
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
||||
get(serve_artifact),
|
||||
)
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
|
||||
post(rearchive_handler),
|
||||
)
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
||||
get(serve_entry_favicon),
|
||||
|
|
@ -836,83 +832,6 @@ async fn get_capture_job_handler(
|
|||
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
||||
}
|
||||
|
||||
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
|
||||
///
|
||||
/// Re-archives an existing tweet or tweet_thread entry in-place:
|
||||
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
|
||||
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
|
||||
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
|
||||
///
|
||||
/// Returns 202 immediately with a job_uid the client should poll via
|
||||
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
|
||||
async fn rearchive_handler(
|
||||
State(state): State<AppState>,
|
||||
auth_user: AuthUser,
|
||||
Path((archive_id, entry_uid)): Path<(String, String)>,
|
||||
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
|
||||
auth_user.require_role(ROLE_USER)?;
|
||||
let mounted = mounted_archive(&state, &archive_id)?;
|
||||
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// Create a capture job record so the client can poll for completion.
|
||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
||||
drop(conn);
|
||||
|
||||
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
|
||||
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
|
||||
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
||||
Err(_) => vec![],
|
||||
};
|
||||
let capture_config = capture::CaptureConfig {
|
||||
cookie_rules,
|
||||
ublock_enabled: None,
|
||||
cookie_ext_enabled: None,
|
||||
reader_mode: false,
|
||||
};
|
||||
|
||||
let job_uid_bg = job_uid.clone();
|
||||
let archive_path = mounted.archive_path.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let conn = match database::open_or_initialize(&archive_path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
|
||||
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
|
||||
Ok(result) => {
|
||||
if result.status == "completed" {
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "completed", None, None, None,
|
||||
).ok();
|
||||
} else {
|
||||
// "not_a_tweet" or "scraper_failed" — surface as a job failure
|
||||
// so the client sees a meaningful error message.
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "failed",
|
||||
None, Some(&result.message), None,
|
||||
).ok();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "failed",
|
||||
None, Some(&format!("{e:#}")), None,
|
||||
).ok();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok((
|
||||
StatusCode::ACCEPTED,
|
||||
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
|
||||
))
|
||||
}
|
||||
|
||||
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
||||
///
|
||||
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
||||
|
|
|
|||
1
crates/archivr-server/static/assets/index-C0BIEdow.css
Normal file
1
crates/archivr-server/static/assets/index-C0BIEdow.css
Normal file
File diff suppressed because one or more lines are too long
40
crates/archivr-server/static/assets/index-D6iX-c2T.js
Normal file
40
crates/archivr-server/static/assets/index-D6iX-c2T.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@
|
|||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Archivr</title>
|
||||
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
|
||||
<script type="module" crossorigin src="/assets/index-D6iX-c2T.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-C0BIEdow.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
|
|
|||
|
|
@ -241,12 +241,10 @@ export default function App() {
|
|||
])
|
||||
}, [archiveId, searchQuery, tagFilter, loadEntries])
|
||||
|
||||
const handleToast = useCallback((text, locator, type = 'error', headline = null) => {
|
||||
// Only suppress per-item ublock/cookie warnings (those carry a locator).
|
||||
// Batch summary warnings (locator = null) must always show.
|
||||
if (type === 'warning' && ublockWarningIgnored && locator) return
|
||||
const handleToast = useCallback((text, locator, type = 'error') => {
|
||||
if (type === 'warning' && ublockWarningIgnored) return
|
||||
const id = ++toastIdRef.current
|
||||
setToasts(prev => [...prev, { id, text, locator, type, headline }])
|
||||
setToasts(prev => [...prev, { id, text, locator, type }])
|
||||
}, [ublockWarningIgnored])
|
||||
|
||||
const handleDismissToast = useCallback((id) => {
|
||||
|
|
@ -256,7 +254,7 @@ export default function App() {
|
|||
const handleIgnoreUblock = useCallback(() => {
|
||||
sessionStorage.setItem('ublockWarningIgnored', 'true')
|
||||
setUblockWarningIgnored(true)
|
||||
setToasts(prev => prev.filter(t => !(t.type === 'warning' && t.locator)))
|
||||
setToasts(prev => prev.filter(t => t.type !== 'warning'))
|
||||
}, [])
|
||||
|
||||
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;
|
||||
|
|
|
|||
|
|
@ -371,18 +371,6 @@ export async function deleteOrphanBlobs(archiveId) {
|
|||
return res.json()
|
||||
}
|
||||
|
||||
// ── Re-archive ────────────────────────────────────────────────────────────────
|
||||
export async function rearchiveEntry(archiveId, entryUid) {
|
||||
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}/rearchive`, {
|
||||
method: 'POST',
|
||||
})
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}))
|
||||
throw new Error(body.message || `rearchive failed: ${res.status}`)
|
||||
}
|
||||
return res.json() // { job_uid, status: 'pending' }
|
||||
}
|
||||
|
||||
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
||||
|
||||
export async function listCookieRules() {
|
||||
|
|
|
|||
|
|
@ -85,8 +85,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
const pollIntervals = useRef(new Map())
|
||||
// itemId → debounce timeoutId for probe calls
|
||||
const probeTimers = useRef(new Map())
|
||||
// batchId → { total, archived, warnings, failed }; only populated for multi-URL submits
|
||||
const batchRef = useRef(new Map())
|
||||
// stable ref so probe callbacks always see the current archiveId
|
||||
const archiveIdRef = useRef(archiveId)
|
||||
useEffect(() => { archiveIdRef.current = archiveId }, [archiveId])
|
||||
|
|
@ -196,7 +194,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
}
|
||||
}, [])
|
||||
|
||||
function startPolling(itemId, jobUid, locator, aid, batchId = null) {
|
||||
function startPolling(itemId, jobUid, locator, aid) {
|
||||
if (pollIntervals.current.has(jobUid)) return // already polling
|
||||
const intervalId = setInterval(async () => {
|
||||
try {
|
||||
|
|
@ -204,6 +202,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
if (updated.status === 'completed') {
|
||||
clearInterval(pollIntervals.current.get(jobUid))
|
||||
pollIntervals.current.delete(jobUid)
|
||||
// Show ✓ briefly then remove the row; if last row add a fresh one
|
||||
setItems(prev => prev.map(it => it.id === itemId ? { ...it, status: 'completed' } : it))
|
||||
setTimeout(() => {
|
||||
setItems(prev => {
|
||||
|
|
@ -212,25 +211,13 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
})
|
||||
}, 1400)
|
||||
onCapturedRef.current()
|
||||
// Warn if uBlock was requested but the extension wasn't available
|
||||
try {
|
||||
const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null
|
||||
if (notes?.ublock_skipped || notes?.cookie_ext_skipped) {
|
||||
const both = notes.ublock_skipped && notes.cookie_ext_skipped
|
||||
const msg = both
|
||||
? 'Captured without ad-blocking or cookie-consent extension. Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'
|
||||
: notes.ublock_skipped
|
||||
? 'Captured without ad-blocking. ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid.'
|
||||
: 'Captured without cookie-consent extension. ARCHIVR_COOKIE_EXT is not set or the path is invalid.'
|
||||
onToastRef.current(msg, locator, 'warning')
|
||||
settleBatch(batchId, 'warning', locator)
|
||||
} else {
|
||||
if (!batchId) onToastRef.current(null, locator, 'success')
|
||||
settleBatch(batchId, 'archived')
|
||||
}
|
||||
} catch {
|
||||
if (!batchId) onToastRef.current(null, locator, 'success')
|
||||
settleBatch(batchId, 'archived')
|
||||
onToastRef.current(null, locator, 'warning')
|
||||
}
|
||||
} catch {}
|
||||
} else if (updated.status === 'failed') {
|
||||
clearInterval(pollIntervals.current.get(jobUid))
|
||||
pollIntervals.current.delete(jobUid)
|
||||
|
|
@ -239,7 +226,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
it.id === itemId ? { ...it, status: 'failed', error: errText } : it
|
||||
))
|
||||
onToastRef.current(errText, locator)
|
||||
settleBatch(batchId, 'failed', locator)
|
||||
}
|
||||
// 'pending' / 'running': keep polling
|
||||
} catch (e) {
|
||||
|
|
@ -250,51 +236,14 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
it.id === itemId ? { ...it, status: 'failed', error: msg } : it
|
||||
))
|
||||
onToastRef.current(msg, locator)
|
||||
settleBatch(batchId, 'failed', locator)
|
||||
}
|
||||
}, 500)
|
||||
pollIntervals.current.set(jobUid, intervalId)
|
||||
}
|
||||
|
||||
// Increments the batch counter for the given outcome and emits a summary
|
||||
// toast once all jobs in the batch have settled.
|
||||
// 'warning' counts as archived (succeeded with caveats); locator recorded for detail text.
|
||||
function settleBatch(batchId, outcome, locator = null) {
|
||||
if (!batchId) return
|
||||
const batch = batchRef.current.get(batchId)
|
||||
if (!batch) return
|
||||
if (outcome === 'archived' || outcome === 'warning') {
|
||||
batch.archived++
|
||||
if (outcome === 'warning') { batch.warnings++; if (locator) batch.warningLocators.push(locator) }
|
||||
} else {
|
||||
batch.failed++
|
||||
if (locator) batch.failedLocators.push(locator)
|
||||
}
|
||||
if (batch.archived + batch.failed < batch.total) return
|
||||
// All settled — build headline + detail text, then emit summary and clean up.
|
||||
batchRef.current.delete(batchId)
|
||||
const { archived, warnings, failed, failedLocators, warningLocators } = batch
|
||||
let headline
|
||||
if (archived === 0) {
|
||||
headline = `${failed} failed`
|
||||
} else {
|
||||
const archivedStr = warnings > 0
|
||||
? `${archived} archived (${warnings} with warnings)`
|
||||
: `${archived} archived`
|
||||
headline = failed > 0 ? `${archivedStr}, ${failed} failed` : archivedStr
|
||||
}
|
||||
const type = archived === 0 ? 'error' : (failed > 0 || warnings > 0) ? 'warning' : 'success'
|
||||
const parts = []
|
||||
if (failedLocators.length > 0) parts.push(`Failed:\n${failedLocators.map(l => ` ${l}`).join('\n')}`)
|
||||
if (warningLocators.length > 0) parts.push(`With warnings:\n${warningLocators.map(l => ` ${l}`).join('\n')}`)
|
||||
const text = parts.length > 0 ? parts.join('\n') : null
|
||||
onToastRef.current(text, null, type, headline)
|
||||
}
|
||||
|
||||
|
||||
async function submitItem(item, batchId = null) {
|
||||
async function submitItem(item) {
|
||||
if (!item.locator.trim()) return
|
||||
const aid = archiveId
|
||||
const aid = archiveId // capture at submit time
|
||||
const loc = item.locator.trim()
|
||||
const qual = item.quality || 'best'
|
||||
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
|
||||
|
|
@ -304,27 +253,17 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
|||
setItems(prev => prev.map(it =>
|
||||
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
|
||||
))
|
||||
startPolling(item.id, job.job_uid, loc, aid, batchId)
|
||||
startPolling(item.id, job.job_uid, loc, aid)
|
||||
} catch (e) {
|
||||
const msg = e.message || 'Submission failed.'
|
||||
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'failed', error: msg } : it))
|
||||
onToastRef.current(msg, loc)
|
||||
settleBatch(batchId, 'failed', loc)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function handleArchive() {
|
||||
const toSubmit = items.filter(it => it.status === 'idle' && it.locator.trim())
|
||||
if (toSubmit.length === 0) return
|
||||
const batchId = toSubmit.length > 1
|
||||
? (crypto.randomUUID?.() ?? `batch-${Date.now()}`)
|
||||
: null
|
||||
if (batchId) {
|
||||
batchRef.current.set(batchId, { total: toSubmit.length, archived: 0, warnings: 0, failed: 0, failedLocators: [], warningLocators: [] })
|
||||
}
|
||||
toSubmit.forEach(it => submitItem(it, batchId))
|
||||
dialogRef.current?.close()
|
||||
toSubmit.forEach(it => submitItem(it))
|
||||
}
|
||||
|
||||
function addRow() {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useState, useEffect, useRef } from 'react'
|
||||
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry, rearchiveEntry, pollCaptureJob } from '../api'
|
||||
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry } from '../api'
|
||||
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
|
||||
|
||||
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
|
||||
|
|
@ -21,14 +21,8 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
const titleCancelRef = useRef(false)
|
||||
const [editingTitle, setEditingTitle] = useState(false)
|
||||
const [titleDraft, setTitleDraft] = useState('')
|
||||
const [rearchiveState, setRearchiveState] = useState('idle') // 'idle' | 'running' | 'done' | 'error'
|
||||
const [rearchiveError, setRearchiveError] = useState('')
|
||||
const rearchivePollRef = useRef(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (rearchivePollRef.current) { clearInterval(rearchivePollRef.current); rearchivePollRef.current = null }
|
||||
setRearchiveState('idle')
|
||||
setRearchiveError('')
|
||||
if (!selectedEntry || !archiveId) {
|
||||
setDetail(null)
|
||||
setTags([])
|
||||
|
|
@ -53,12 +47,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
}).catch(() => {})
|
||||
}, [selectedEntry, archiveId])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
}
|
||||
}, [])
|
||||
|
||||
async function handleTitleSave() {
|
||||
const newTitle = titleDraft.trim() || null
|
||||
try {
|
||||
|
|
@ -109,46 +97,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
}
|
||||
}
|
||||
|
||||
async function handleRearchive() {
|
||||
if (!selectedEntry || !archiveId || rearchiveState === 'running') return
|
||||
setRearchiveState('running')
|
||||
setRearchiveError('')
|
||||
try {
|
||||
const { job_uid } = await rearchiveEntry(archiveId, selectedEntry.entry_uid)
|
||||
// Poll for job completion at 500ms — same interval as CaptureDialog
|
||||
rearchivePollRef.current = setInterval(async () => {
|
||||
try {
|
||||
const job = await pollCaptureJob(archiveId, job_uid)
|
||||
if (job.status === 'completed') {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
rearchivePollRef.current = null
|
||||
setRearchiveState('done')
|
||||
// Refresh entry detail so artifact list updates
|
||||
const [det, tgs] = await Promise.all([
|
||||
fetchEntryDetail(archiveId, selectedEntry.entry_uid),
|
||||
fetchEntryTags(archiveId, selectedEntry.entry_uid),
|
||||
])
|
||||
setDetail(det)
|
||||
setTags(tgs)
|
||||
} else if (job.status === 'failed') {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
rearchivePollRef.current = null
|
||||
setRearchiveState('error')
|
||||
setRearchiveError(job.error_text || 'Re-archive failed.')
|
||||
}
|
||||
} catch {
|
||||
clearInterval(rearchivePollRef.current)
|
||||
rearchivePollRef.current = null
|
||||
setRearchiveState('error')
|
||||
setRearchiveError('Network error while polling.')
|
||||
}
|
||||
}, 500)
|
||||
} catch (e) {
|
||||
setRearchiveState('error')
|
||||
setRearchiveError(e.message || 'Failed to start re-archive.')
|
||||
}
|
||||
}
|
||||
|
||||
const metaRows = detail ? [
|
||||
['Added', formatTimestamp(detail.summary.archived_at)],
|
||||
['Source', detail.summary.source_kind],
|
||||
|
|
@ -296,25 +244,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
|||
</div>
|
||||
)}
|
||||
|
||||
{detail && (detail.summary.entity_kind === 'tweet' || detail.summary.entity_kind === 'tweet_thread') && (
|
||||
<div className="rail-section">
|
||||
<div className="rail-section-heading">Actions</div>
|
||||
<button
|
||||
className="rail-rearchive-btn"
|
||||
onClick={handleRearchive}
|
||||
disabled={rearchiveState === 'running'}
|
||||
>
|
||||
{rearchiveState === 'running' ? 'Re-archiving\u2026' : 'Re-archive'}
|
||||
</button>
|
||||
{rearchiveState === 'done' && (
|
||||
<p className="form-msg form-msg--ok" style={{ marginTop: '6px' }}>Re-archived successfully.</p>
|
||||
)}
|
||||
{rearchiveState === 'error' && (
|
||||
<p className="form-msg form-msg--err" style={{ marginTop: '6px' }}>{rearchiveError}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="rail-delete-zone">
|
||||
<button className="rail-delete-btn" onClick={handleDeleteEntry}>
|
||||
Delete entry
|
||||
|
|
|
|||
|
|
@ -13,27 +13,9 @@ export default function ToastStack({ toasts, onDismiss, onIgnoreUblock }) {
|
|||
)
|
||||
}
|
||||
|
||||
// For URLs: show hostname/…/tail (drops protocol + middle path, keeps domain + identifier).
|
||||
// For shorthands (yt:, x:, etc.): keep the tail since it's the whole identifier.
|
||||
// CSS text-overflow:ellipsis on .toast-locator handles any remaining overflow.
|
||||
function shortLocator(locator) {
|
||||
if (!locator) return null
|
||||
if (locator.length <= 52) return locator
|
||||
try {
|
||||
const { hostname, pathname, search } = new URL(locator)
|
||||
const segments = pathname.split('/').filter(Boolean)
|
||||
const tail = (segments[segments.length - 1] ?? '') + search
|
||||
const candidate = tail ? `${hostname}/\u2026/${tail}` : hostname
|
||||
return candidate.length <= 56 ? candidate : candidate.slice(0, 53) + '\u2026'
|
||||
} catch {
|
||||
return '\u2026' + locator.slice(-51)
|
||||
}
|
||||
}
|
||||
|
||||
function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||
const [expanded, setExpanded] = useState(false)
|
||||
const isWarning = toast.type === 'warning'
|
||||
const isSuccess = toast.type === 'success'
|
||||
|
||||
// Auto-dismiss after TTL; paused while detail is expanded
|
||||
useEffect(() => {
|
||||
|
|
@ -42,24 +24,9 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
return () => clearTimeout(timer)
|
||||
}, [expanded, toast.id, onDismiss])
|
||||
|
||||
const short = shortLocator(toast.locator)
|
||||
|
||||
if (isSuccess) {
|
||||
return (
|
||||
<div className="toast toast--success" role="alert" aria-atomic="true">
|
||||
<div className="toast-top">
|
||||
<span className="toast-icon" aria-hidden="true">✓</span>
|
||||
<div className="toast-body">
|
||||
<span className="toast-headline">{toast.headline || 'Archived'}</span>
|
||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
||||
</div>
|
||||
<div className="toast-btns">
|
||||
<button type="button" className="toast-dismiss" onClick={() => onDismiss(toast.id)} aria-label="Dismiss">×</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
const short = toast.locator
|
||||
? (toast.locator.length > 48 ? toast.locator.slice(0, 45) + '\u2026' : toast.locator)
|
||||
: null
|
||||
|
||||
if (isWarning) {
|
||||
return (
|
||||
|
|
@ -67,11 +34,10 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
<div className="toast-top">
|
||||
<span className="toast-icon" aria-hidden="true">⚠</span>
|
||||
<div className="toast-body">
|
||||
<span className="toast-headline">{toast.headline || 'Archived with warnings'}</span>
|
||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
||||
<span className="toast-headline">Ad-blocking unavailable</span>
|
||||
{short && <span className="toast-locator">{short}</span>}
|
||||
</div>
|
||||
<div className="toast-btns">
|
||||
{toast.text && (
|
||||
<button
|
||||
type="button"
|
||||
className="toast-view-btn"
|
||||
|
|
@ -80,8 +46,6 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
>
|
||||
{expanded ? 'Hide' : 'Details'}
|
||||
</button>
|
||||
)}
|
||||
{toast.locator && (
|
||||
<button
|
||||
type="button"
|
||||
className="toast-view-btn toast-ignore-btn"
|
||||
|
|
@ -89,7 +53,6 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
>
|
||||
Ignore
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
className="toast-dismiss"
|
||||
|
|
@ -102,7 +65,7 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
</div>
|
||||
{expanded && (
|
||||
<p className="toast-warning-detail">
|
||||
{toast.text || 'The page was captured but one or more browser extensions were unavailable (ad-blocking or cookie-consent). Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'}
|
||||
{toast.text || 'ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid. The page was captured without ad-blocking. Set ARCHIVR_UBLOCK_EXT to the unpacked uBlock Origin Lite extension directory to enable ad-blocking, or set ARCHIVR_UBLOCK=false to silence this warning.'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -114,8 +77,8 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
|||
<div className="toast-top">
|
||||
<span className="toast-icon" aria-hidden="true">✕</span>
|
||||
<div className="toast-body">
|
||||
<span className="toast-headline">{toast.headline || 'Capture failed'}</span>
|
||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
||||
<span className="toast-headline">Capture failed</span>
|
||||
{short && <span className="toast-locator">{short}</span>}
|
||||
</div>
|
||||
<div className="toast-btns">
|
||||
{toast.text && (
|
||||
|
|
|
|||
|
|
@ -552,24 +552,6 @@ select {
|
|||
border-color: var(--accent);
|
||||
}
|
||||
|
||||
.rail-rearchive-btn {
|
||||
width: 100%;
|
||||
padding: 6px 12px;
|
||||
background: var(--surface-2, #2a2a2a);
|
||||
color: var(--text, #e0e0e0);
|
||||
border: 1px solid var(--border, #444);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
}
|
||||
.rail-rearchive-btn:hover:not(:disabled) {
|
||||
background: var(--surface-3, #333);
|
||||
}
|
||||
.rail-rearchive-btn:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* ── Capture dialog ──────────────────────────────────────────────────────── */
|
||||
.capture-dialog {
|
||||
border: 1px solid var(--line);
|
||||
|
|
@ -887,7 +869,6 @@ select {
|
|||
}
|
||||
.toast--error { border-left: 3px solid var(--accent); }
|
||||
.toast--warning { border-left: 3px solid #e8a000; }
|
||||
.toast--success { border-left: 3px solid #22a855; }
|
||||
|
||||
.toast-top {
|
||||
display: flex;
|
||||
|
|
@ -902,8 +883,6 @@ select {
|
|||
color: var(--accent);
|
||||
margin-top: 2px;
|
||||
}
|
||||
.toast--success .toast-icon { color: #22a855; }
|
||||
.toast--warning .toast-icon { color: #e8a000; }
|
||||
.toast-body {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue