mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
Compare commits
No commits in common. "2779afee2dd36271c20a5384c98cfc71f0220885" and "2e8820a0dac027f70bcd364877b880111d0ab8ec" have entirely different histories.
2779afee2d
...
2e8820a0da
16 changed files with 215 additions and 675 deletions
|
|
@ -742,52 +742,6 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
|
|
||||||
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
|
|
||||||
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
|
|
||||||
fn register_tweet_artifacts(
|
|
||||||
conn: &rusqlite::Connection,
|
|
||||||
store_path: &Path,
|
|
||||||
entry_id: i64,
|
|
||||||
tweet_json_relpaths: &[String],
|
|
||||||
) -> Result<()> {
|
|
||||||
for relpath in tweet_json_relpaths {
|
|
||||||
database::add_entry_artifact(
|
|
||||||
conn,
|
|
||||||
&database::NewArtifact {
|
|
||||||
entry_id,
|
|
||||||
artifact_role: "raw_tweet_json".to_string(),
|
|
||||||
storage_area: "raw_tweets".to_string(),
|
|
||||||
relpath: relpath.clone(),
|
|
||||||
blob_id: None,
|
|
||||||
logical_path: None,
|
|
||||||
metadata_json: None,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
let json_path = store_path.join(relpath);
|
|
||||||
let json_str = fs::read_to_string(&json_path)
|
|
||||||
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
|
|
||||||
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
|
|
||||||
let raw_path = PathBuf::from(&raw_relpath);
|
|
||||||
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
|
||||||
let blob_id = database::upsert_blob(conn, &blob)?;
|
|
||||||
database::add_entry_artifact(
|
|
||||||
conn,
|
|
||||||
&database::NewArtifact {
|
|
||||||
entry_id,
|
|
||||||
artifact_role: role,
|
|
||||||
storage_area: "raw".to_string(),
|
|
||||||
relpath: raw_relpath,
|
|
||||||
blob_id: Some(blob_id),
|
|
||||||
logical_path: None,
|
|
||||||
metadata_json: None,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn record_tweet_entry(
|
fn record_tweet_entry(
|
||||||
conn: &rusqlite::Connection,
|
conn: &rusqlite::Connection,
|
||||||
store_path: &Path,
|
store_path: &Path,
|
||||||
|
|
@ -797,7 +751,6 @@ fn record_tweet_entry(
|
||||||
requested_locator: &str,
|
requested_locator: &str,
|
||||||
source: Source,
|
source: Source,
|
||||||
tweet_id: &str,
|
tweet_id: &str,
|
||||||
tweet_json_relpaths: &[String],
|
|
||||||
) -> Result<database::ArchivedEntry> {
|
) -> Result<database::ArchivedEntry> {
|
||||||
debug_assert!(run.run_uid.starts_with("run_"));
|
debug_assert!(run.run_uid.starts_with("run_"));
|
||||||
debug_assert!(item.item_uid.starts_with("item_"));
|
debug_assert!(item.item_uid.starts_with("item_"));
|
||||||
|
|
@ -811,7 +764,7 @@ fn record_tweet_entry(
|
||||||
Some(&canonical_locator),
|
Some(&canonical_locator),
|
||||||
&canonical_locator,
|
&canonical_locator,
|
||||||
)?;
|
)?;
|
||||||
// Read the primary tweet JSON to extract title before entry creation.
|
// Read tweet JSON early to extract title before entry creation
|
||||||
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
||||||
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
|
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
|
||||||
let tweet_meta = tweet_metadata_from_json(&tweet_json);
|
let tweet_meta = tweet_metadata_from_json(&tweet_json);
|
||||||
|
|
@ -841,8 +794,36 @@ fn record_tweet_entry(
|
||||||
)?;
|
)?;
|
||||||
create_structured_root(store_path, &entry)?;
|
create_structured_root(store_path, &entry)?;
|
||||||
|
|
||||||
// Register all tweet JSONs and their blobs.
|
database::add_entry_artifact(
|
||||||
register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?;
|
conn,
|
||||||
|
&database::NewArtifact {
|
||||||
|
entry_id: entry.id,
|
||||||
|
artifact_role: "raw_tweet_json".to_string(),
|
||||||
|
storage_area: "raw_tweets".to_string(),
|
||||||
|
relpath: path_to_store_string(&tweet_json_relpath),
|
||||||
|
blob_id: None,
|
||||||
|
logical_path: None,
|
||||||
|
metadata_json: None,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
|
||||||
|
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
|
||||||
|
let raw_path = PathBuf::from(&raw_relpath);
|
||||||
|
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
|
||||||
|
let blob_id = database::upsert_blob(conn, &blob)?;
|
||||||
|
database::add_entry_artifact(
|
||||||
|
conn,
|
||||||
|
&database::NewArtifact {
|
||||||
|
entry_id: entry.id,
|
||||||
|
artifact_role: role,
|
||||||
|
storage_area: "raw".to_string(),
|
||||||
|
relpath: raw_relpath,
|
||||||
|
blob_id: Some(blob_id),
|
||||||
|
logical_path: None,
|
||||||
|
metadata_json: None,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
database::complete_archive_run_item(conn, item.id, entry.id)?;
|
database::complete_archive_run_item(conn, item.id, entry.id)?;
|
||||||
Ok(entry)
|
Ok(entry)
|
||||||
|
|
@ -919,12 +900,14 @@ pub fn perform_capture(
|
||||||
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
|
Ok(downloader::http::UrlKind::Html) => source = Source::WebPage,
|
||||||
Ok(downloader::http::UrlKind::File) => {}
|
Ok(downloader::http::UrlKind::File) => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Probe failed (e.g. Cloudflare JS challenge, 403, network
|
// Record a failed item using the pre-probe source_kind so
|
||||||
// issue). Fall back to treating the URL as an HTML page so
|
// failed_count increments and the item carries error_text.
|
||||||
// that the SingleFile/Chromium path can try — a real browser
|
let (probe_sk, probe_ek, _) = source_metadata(Source::Url);
|
||||||
// can pass bot challenges that a plain HTTP client cannot.
|
let item = database::create_archive_run_item(
|
||||||
eprintln!("warn: probe failed for {locator}, assuming HTML: {e}");
|
&conn, run.id, None, 0, locator, None, probe_sk, probe_ek,
|
||||||
source = Source::WebPage;
|
)?;
|
||||||
|
let msg = format!("Failed to probe URL: {e}");
|
||||||
|
return Err(fail_run(&conn, &run, &item, &msg));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1203,7 +1186,7 @@ pub fn perform_capture(
|
||||||
×tamp,
|
×tamp,
|
||||||
&tweet_cookies,
|
&tweet_cookies,
|
||||||
) {
|
) {
|
||||||
Ok(tweet_json_relpaths) => {
|
Ok(_) => {
|
||||||
let tweet_entry = record_tweet_entry(
|
let tweet_entry = record_tweet_entry(
|
||||||
&conn,
|
&conn,
|
||||||
store_path,
|
store_path,
|
||||||
|
|
@ -1213,7 +1196,6 @@ pub fn perform_capture(
|
||||||
locator,
|
locator,
|
||||||
source,
|
source,
|
||||||
&tweet_id,
|
&tweet_id,
|
||||||
&tweet_json_relpaths,
|
|
||||||
)?;
|
)?;
|
||||||
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
|
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
|
||||||
database::finish_archive_run(&conn, run.id)?;
|
database::finish_archive_run(&conn, run.id)?;
|
||||||
|
|
@ -1382,107 +1364,6 @@ pub fn perform_capture(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Result of a tweet re-archive operation.
|
|
||||||
#[derive(Debug, serde::Serialize)]
|
|
||||||
pub struct RearchiveResult {
|
|
||||||
/// One of: "completed", "not_a_tweet", "scraper_failed".
|
|
||||||
pub status: String,
|
|
||||||
/// Human-readable detail (empty string for "completed").
|
|
||||||
pub message: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Re-archives an existing tweet or tweet_thread entry in-place.
|
|
||||||
///
|
|
||||||
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
|
|
||||||
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
|
|
||||||
/// returns `scraper_failed` WITHOUT touching the existing data.
|
|
||||||
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
|
|
||||||
/// and their media blobs. Does not change archived_at, title, tags, or collections.
|
|
||||||
pub fn perform_rearchive(
|
|
||||||
archive_paths: &ArchivePaths,
|
|
||||||
entry_uid: &str,
|
|
||||||
config: &CaptureConfig,
|
|
||||||
) -> Result<RearchiveResult> {
|
|
||||||
let store_path = &archive_paths.store_path;
|
|
||||||
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
|
|
||||||
|
|
||||||
// Look up the entry.
|
|
||||||
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
|
|
||||||
None => {
|
|
||||||
return Ok(RearchiveResult {
|
|
||||||
status: "not_a_tweet".to_string(),
|
|
||||||
message: "entry not found".to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Some(e) => e,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Only tweet / tweet_thread entries can be re-archived this way.
|
|
||||||
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
|
|
||||||
return Ok(RearchiveResult {
|
|
||||||
status: "not_a_tweet".to_string(),
|
|
||||||
message: format!("entry is '{}', not a tweet", entry.entity_kind),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse source_metadata_json for tweet_id and requested_locator.
|
|
||||||
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
|
|
||||||
.unwrap_or(serde_json::Value::Object(Default::default()));
|
|
||||||
let requested_locator = meta["requested_locator"]
|
|
||||||
.as_str()
|
|
||||||
.unwrap_or("")
|
|
||||||
.to_string();
|
|
||||||
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
|
|
||||||
if requested_locator.is_empty() || tweet_id.is_empty() {
|
|
||||||
return Ok(RearchiveResult {
|
|
||||||
status: "not_a_tweet".to_string(),
|
|
||||||
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let is_thread = entry.entity_kind == "tweet_thread";
|
|
||||||
let timestamp = format!(
|
|
||||||
"{}-{}",
|
|
||||||
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
|
|
||||||
uuid::Uuid::new_v4().simple(),
|
|
||||||
);
|
|
||||||
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
|
|
||||||
|
|
||||||
// Run the scraper (staged). If this fails, existing data is untouched.
|
|
||||||
let tweet_json_relpaths = match downloader::tweets::rearchive(
|
|
||||||
&requested_locator,
|
|
||||||
is_thread,
|
|
||||||
store_path,
|
|
||||||
×tamp,
|
|
||||||
&tweet_cookies,
|
|
||||||
) {
|
|
||||||
Ok(relpaths) => relpaths,
|
|
||||||
Err(e) => {
|
|
||||||
return Ok(RearchiveResult {
|
|
||||||
status: "scraper_failed".to_string(),
|
|
||||||
message: format!("{e:#}"),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Atomically swap artifact rows: delete old, insert new.
|
|
||||||
{
|
|
||||||
let tx = conn.transaction()?;
|
|
||||||
database::delete_entry_artifacts(&tx, entry.id)?;
|
|
||||||
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
|
|
||||||
tx.commit()?;
|
|
||||||
}
|
|
||||||
|
|
||||||
database::refresh_entry_cached_bytes(&conn, entry.id)?;
|
|
||||||
|
|
||||||
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
|
|
||||||
|
|
||||||
Ok(RearchiveResult {
|
|
||||||
status: "completed".to_string(),
|
|
||||||
message: String::new(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mime_for_font_ext(ext: &str) -> Option<String> {
|
fn mime_for_font_ext(ext: &str) -> Option<String> {
|
||||||
match ext {
|
match ext {
|
||||||
".woff2" => Some("font/woff2".to_string()),
|
".woff2" => Some("font/woff2".to_string()),
|
||||||
|
|
@ -2109,7 +1990,6 @@ mod tests {
|
||||||
"tweet:123",
|
"tweet:123",
|
||||||
Source::Tweet,
|
Source::Tweet,
|
||||||
"123",
|
"123",
|
||||||
&["raw_tweets/tweet-123.json".to_string()],
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
database::finish_archive_run(&conn, run.id).unwrap();
|
database::finish_archive_run(&conn, run.id).unwrap();
|
||||||
|
|
|
||||||
|
|
@ -2391,43 +2391,6 @@ fn humanize_slug(slug: &str) -> String {
|
||||||
.join(" ")
|
.join(" ")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A minimal view of an archived entry needed for re-archiving.
|
|
||||||
pub struct EntryForRearchive {
|
|
||||||
pub id: i64,
|
|
||||||
pub entity_kind: String,
|
|
||||||
pub source_metadata_json: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Looks up an entry by its public UID for re-archive purposes.
|
|
||||||
pub fn get_entry_for_rearchive(
|
|
||||||
conn: &Connection,
|
|
||||||
entry_uid: &str,
|
|
||||||
) -> Result<Option<EntryForRearchive>> {
|
|
||||||
conn.query_row(
|
|
||||||
"SELECT id, entity_kind, source_metadata_json \
|
|
||||||
FROM archived_entries WHERE entry_uid = ?1",
|
|
||||||
[entry_uid],
|
|
||||||
|row| {
|
|
||||||
Ok(EntryForRearchive {
|
|
||||||
id: row.get(0)?,
|
|
||||||
entity_kind: row.get(1)?,
|
|
||||||
source_metadata_json: row.get(2)?,
|
|
||||||
})
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.optional()
|
|
||||||
.map_err(anyhow::Error::from)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Deletes all `entry_artifacts` rows for the given entry.
|
|
||||||
/// Call within a transaction for atomicity with re-insertion.
|
|
||||||
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
|
|
||||||
Ok(conn.execute(
|
|
||||||
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
|
|
||||||
[entry_id],
|
|
||||||
)?)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ pub enum UrlKind {
|
||||||
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
|
pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<UrlKind> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder()
|
||||||
.redirect(reqwest::redirect::Policy::limited(10))
|
.redirect(reqwest::redirect::Policy::limited(10))
|
||||||
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
|
.user_agent("archivr/0.1")
|
||||||
.build()
|
.build()
|
||||||
.context("failed to build HTTP client")?;
|
.context("failed to build HTTP client")?;
|
||||||
|
|
||||||
|
|
@ -86,7 +86,7 @@ pub fn probe_url_kind(url: &str, cookies: &HashMap<String, String>) -> Result<Ur
|
||||||
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
|
pub fn download(url: &str, store_path: &Path, timestamp: &str, cookies: &HashMap<String, String>) -> Result<(String, String, Option<String>)> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder()
|
||||||
.redirect(reqwest::redirect::Policy::limited(10))
|
.redirect(reqwest::redirect::Policy::limited(10))
|
||||||
.user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 archivr/0.1")
|
.user_agent("archivr/0.1")
|
||||||
.build()
|
.build()
|
||||||
.context("failed to build HTTP client")?;
|
.context("failed to build HTTP client")?;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::{HashMap, HashSet},
|
||||||
env,
|
env,
|
||||||
ffi::OsString,
|
ffi::OsString,
|
||||||
fs,
|
fs,
|
||||||
|
|
@ -59,39 +59,63 @@ fn build_scraper_args(
|
||||||
args
|
args
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs
|
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
|
||||||
/// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`).
|
///
|
||||||
/// The staging dir starts empty so all files found there are exactly the touched set.
|
/// Invokes the Python scraper, then moves all produced media assets into the
|
||||||
fn run_scraper_staged(
|
/// content-addressed raw store and rewrites the JSON output to use the new
|
||||||
tweet_id: &str,
|
/// store-relative paths. Returns `true` if new content was archived, `false`
|
||||||
|
/// if the tweet was already present and `thread` is `false`.
|
||||||
|
///
|
||||||
|
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
||||||
|
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
||||||
|
pub fn archive(
|
||||||
|
path: &str,
|
||||||
thread: bool,
|
thread: bool,
|
||||||
store_path: &Path,
|
store_path: &Path,
|
||||||
timestamp: &str,
|
timestamp: &str,
|
||||||
cookies: &HashMap<String, String>,
|
cookies: &HashMap<String, String>,
|
||||||
) -> Result<Vec<String>> {
|
) -> Result<bool> {
|
||||||
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
||||||
|
// Output directory for Tweet JSON files.
|
||||||
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
|
|
||||||
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
|
|
||||||
// Scraper media-dir = staging_dir/media/
|
|
||||||
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
|
|
||||||
let staging_tweets_dir = staging_dir.join("raw_tweets");
|
|
||||||
fs::create_dir_all(&staging_tweets_dir)?;
|
|
||||||
|
|
||||||
// Final destination for tweet JSONs.
|
|
||||||
let output_dir = store_path.join("raw_tweets");
|
let output_dir = store_path.join("raw_tweets");
|
||||||
|
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
|
||||||
|
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
|
||||||
|
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||||
|
|
||||||
fs::create_dir_all(&output_dir)?;
|
fs::create_dir_all(&output_dir)?;
|
||||||
|
fs::create_dir_all(&temp_dir)?;
|
||||||
|
|
||||||
|
// Path to the root - the to-be-archived tweet's JSON file.
|
||||||
|
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
|
||||||
|
if !thread && root_json.exists() {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let before = tweet_json_files(&output_dir)?;
|
||||||
|
|
||||||
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
||||||
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
||||||
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
||||||
|
|
||||||
|
// Credentials: only use cookie rules as Twitter credentials when the resolved
|
||||||
|
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
|
||||||
|
// requires. A global cookie rule for an unrelated site must not suppress the
|
||||||
|
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
|
||||||
|
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
|
||||||
|
// The temp file is written in the semicolon-delimited format the Python scraper
|
||||||
|
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
|
||||||
|
// subprocess returns so secrets are never left on disk on failure.
|
||||||
let temp_creds_path: Option<PathBuf>;
|
let temp_creds_path: Option<PathBuf>;
|
||||||
let credentials_file: PathBuf;
|
let credentials_file: PathBuf;
|
||||||
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
|
||||||
|
let has_twitter_cookies =
|
||||||
|
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
|
||||||
|
|
||||||
if has_twitter_cookies {
|
if has_twitter_cookies {
|
||||||
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
|
||||||
|
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
|
||||||
let creds_str = cookies
|
let creds_str = cookies
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(k, v)| format!("{k}={v}"))
|
.map(|(k, v)| format!("{k}={v}"))
|
||||||
|
|
@ -111,7 +135,7 @@ fn run_scraper_staged(
|
||||||
let mut f = std::fs::File::create(&cf)
|
let mut f = std::fs::File::create(&cf)
|
||||||
.context("failed to write twitter credentials file")?;
|
.context("failed to write twitter credentials file")?;
|
||||||
f.write_all(creds_str.as_bytes())
|
f.write_all(creds_str.as_bytes())
|
||||||
.context("failed to write twitter credentials file")?
|
.context("failed to write twitter credentials file")?;
|
||||||
}
|
}
|
||||||
temp_creds_path = Some(cf.clone());
|
temp_creds_path = Some(cf.clone());
|
||||||
credentials_file = cf;
|
credentials_file = cf;
|
||||||
|
|
@ -132,21 +156,28 @@ fn run_scraper_staged(
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut cmd = Command::new(&python);
|
let mut cmd = Command::new(&python);
|
||||||
// Run the scraper from staging_dir so relative paths in JSON resolve correctly.
|
cmd.current_dir(&temp_dir).arg(&scraper_path);
|
||||||
cmd.current_dir(&staging_dir).arg(&scraper_path);
|
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
|
||||||
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
|
|
||||||
cmd.arg(arg);
|
cmd.arg(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Hold the Result so we can delete the credentials file before propagating
|
||||||
|
// any error — including spawn failures.
|
||||||
let spawn_result = cmd.output().with_context(|| {
|
let spawn_result = cmd.output().with_context(|| {
|
||||||
format!("Failed to spawn tweet scraper at {}", scraper_path.display())
|
format!(
|
||||||
|
"Failed to spawn tweet scraper at {}",
|
||||||
|
scraper_path.display()
|
||||||
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Remove temp credentials file unconditionally — secrets must not persist.
|
||||||
if let Some(cf) = &temp_creds_path {
|
if let Some(cf) = &temp_creds_path {
|
||||||
let _ = fs::remove_file(cf);
|
let _ = fs::remove_file(cf);
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = spawn_result?;
|
let output = spawn_result?;
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let _ = fs::remove_dir_all(&staging_dir);
|
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
bail!(
|
bail!(
|
||||||
|
|
@ -156,9 +187,7 @@ fn run_scraper_staged(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
|
|
||||||
if !root_json.exists() {
|
if !root_json.exists() {
|
||||||
let _ = fs::remove_dir_all(&staging_dir);
|
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
bail!(
|
bail!(
|
||||||
|
|
@ -169,82 +198,13 @@ fn run_scraper_staged(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove the scraping_summary.json if the scraper left one.
|
cleanup_summary(&output_dir)?;
|
||||||
cleanup_summary(&staging_tweets_dir)?;
|
let after = tweet_json_files(&output_dir)?;
|
||||||
|
let new_jsons = new_tweet_jsons(&before, &after);
|
||||||
|
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
|
||||||
|
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
|
||||||
|
|
||||||
// Collect all tweet-*.json files from staging (this is the exact touched set).
|
Ok(true)
|
||||||
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
|
|
||||||
.filter_map(|e| e.ok())
|
|
||||||
.map(|e| e.path())
|
|
||||||
.filter(|p| {
|
|
||||||
p.file_name()
|
|
||||||
.and_then(|n| n.to_str())
|
|
||||||
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
staged_jsons.sort();
|
|
||||||
|
|
||||||
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
|
|
||||||
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
|
|
||||||
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
|
|
||||||
|
|
||||||
// Move each staged JSON to its final destination, collecting store-relative relpaths.
|
|
||||||
let mut relpaths = Vec::with_capacity(staged_jsons.len());
|
|
||||||
for staged_path in &staged_jsons {
|
|
||||||
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
|
|
||||||
let dest = output_dir.join(filename);
|
|
||||||
fs::rename(staged_path, &dest)
|
|
||||||
.or_else(|_| {
|
|
||||||
fs::copy(staged_path, &dest).map(|_| ())?;
|
|
||||||
fs::remove_file(staged_path)
|
|
||||||
})
|
|
||||||
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
|
|
||||||
let rel = dest
|
|
||||||
.strip_prefix(store_path)
|
|
||||||
.context("dest tweet JSON is not under store_path")?;
|
|
||||||
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
|
|
||||||
let _ = fs::remove_dir_all(&staging_dir);
|
|
||||||
|
|
||||||
Ok(relpaths)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Archives a tweet (or full thread) identified by `path`.
|
|
||||||
///
|
|
||||||
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
|
|
||||||
/// For single tweets already on disk, returns the existing file path without re-downloading.
|
|
||||||
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
|
|
||||||
pub fn archive(
|
|
||||||
path: &str,
|
|
||||||
thread: bool,
|
|
||||||
store_path: &Path,
|
|
||||||
timestamp: &str,
|
|
||||||
cookies: &HashMap<String, String>,
|
|
||||||
) -> Result<Vec<String>> {
|
|
||||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
|
||||||
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
|
|
||||||
if !thread && root_json.exists() {
|
|
||||||
// Already present; skip re-download, return the existing file for artifact registration.
|
|
||||||
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
|
|
||||||
}
|
|
||||||
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
|
|
||||||
/// Always runs the scraper; output is staged before replacing existing files.
|
|
||||||
/// Returns store-relative relpaths of all produced tweet JSON files.
|
|
||||||
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
|
|
||||||
pub fn rearchive(
|
|
||||||
path: &str,
|
|
||||||
thread: bool,
|
|
||||||
store_path: &Path,
|
|
||||||
timestamp: &str,
|
|
||||||
cookies: &HashMap<String, String>,
|
|
||||||
) -> Result<Vec<String>> {
|
|
||||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
|
||||||
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
||||||
|
|
@ -256,6 +216,33 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the set of `tweet-*.json` files present in `output_dir`.
|
||||||
|
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
|
||||||
|
let mut files = HashSet::new();
|
||||||
|
|
||||||
|
for entry in fs::read_dir(output_dir)? {
|
||||||
|
let entry = entry?;
|
||||||
|
let path = entry.path();
|
||||||
|
|
||||||
|
if path.is_file()
|
||||||
|
&& path
|
||||||
|
.file_name()
|
||||||
|
.and_then(|name| name.to_str())
|
||||||
|
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
|
||||||
|
{
|
||||||
|
files.insert(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(files)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the sorted list of JSON files present in `after` but not in `before`.
|
||||||
|
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
|
||||||
|
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
|
||||||
|
files.sort();
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
||||||
fn avatar_regex() -> &'static Regex {
|
fn avatar_regex() -> &'static Regex {
|
||||||
|
|
@ -532,9 +519,9 @@ mod tests {
|
||||||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||||
|
|
||||||
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||||
|
|
||||||
assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]);
|
assert!(!archived);
|
||||||
|
|
||||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||||
let _ = fs::remove_dir_all(store_path);
|
let _ = fs::remove_dir_all(store_path);
|
||||||
|
|
@ -589,7 +576,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
|
||||||
{
|
{
|
||||||
"id": "$tweet_id",
|
"id": "$tweet_id",
|
||||||
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
||||||
"author": { "avatar_local_path": "$media_dir/avatars/author.jpg" }
|
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
"#,
|
"#,
|
||||||
|
|
@ -605,16 +592,16 @@ EOF
|
||||||
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
||||||
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
||||||
|
|
||||||
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
|
||||||
let tweet_file = output_dir.join("tweet-123.json");
|
let tweet_file = output_dir.join("tweet-123.json");
|
||||||
let contents = fs::read_to_string(&tweet_file).unwrap();
|
let contents = fs::read_to_string(&tweet_file).unwrap();
|
||||||
|
|
||||||
assert!(!relpaths.is_empty());
|
assert!(archived);
|
||||||
assert!(tweet_file.exists());
|
assert!(tweet_file.exists());
|
||||||
assert!(!output_dir.join("scraping_summary.json").exists());
|
assert!(!output_dir.join("scraping_summary.json").exists());
|
||||||
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
||||||
assert!(contents.contains(r#""local_path": "raw/"#));
|
assert!(contents.contains(r#""local_path": "raw/"#));
|
||||||
assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists());
|
assert!(!store_path.join("temp").join("ts").exists());
|
||||||
|
|
||||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||||
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
||||||
|
|
|
||||||
|
|
@ -221,10 +221,6 @@ pub fn app_with_state(state: AppState) -> Router {
|
||||||
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
||||||
get(serve_artifact),
|
get(serve_artifact),
|
||||||
)
|
)
|
||||||
.route(
|
|
||||||
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
|
|
||||||
post(rearchive_handler),
|
|
||||||
)
|
|
||||||
.route(
|
.route(
|
||||||
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
||||||
get(serve_entry_favicon),
|
get(serve_entry_favicon),
|
||||||
|
|
@ -836,83 +832,6 @@ async fn get_capture_job_handler(
|
||||||
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
|
|
||||||
///
|
|
||||||
/// Re-archives an existing tweet or tweet_thread entry in-place:
|
|
||||||
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
|
|
||||||
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
|
|
||||||
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
|
|
||||||
///
|
|
||||||
/// Returns 202 immediately with a job_uid the client should poll via
|
|
||||||
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
|
|
||||||
async fn rearchive_handler(
|
|
||||||
State(state): State<AppState>,
|
|
||||||
auth_user: AuthUser,
|
|
||||||
Path((archive_id, entry_uid)): Path<(String, String)>,
|
|
||||||
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
|
|
||||||
auth_user.require_role(ROLE_USER)?;
|
|
||||||
let mounted = mounted_archive(&state, &archive_id)?;
|
|
||||||
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
|
|
||||||
.map_err(ApiError::from)?;
|
|
||||||
|
|
||||||
// Create a capture job record so the client can poll for completion.
|
|
||||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
|
||||||
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
|
||||||
drop(conn);
|
|
||||||
|
|
||||||
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
|
|
||||||
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
|
|
||||||
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
|
||||||
Err(_) => vec![],
|
|
||||||
};
|
|
||||||
let capture_config = capture::CaptureConfig {
|
|
||||||
cookie_rules,
|
|
||||||
ublock_enabled: None,
|
|
||||||
cookie_ext_enabled: None,
|
|
||||||
reader_mode: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
let job_uid_bg = job_uid.clone();
|
|
||||||
let archive_path = mounted.archive_path.clone();
|
|
||||||
tokio::task::spawn_blocking(move || {
|
|
||||||
let conn = match database::open_or_initialize(&archive_path) {
|
|
||||||
Ok(c) => c,
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
|
|
||||||
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
|
|
||||||
Ok(result) => {
|
|
||||||
if result.status == "completed" {
|
|
||||||
database::update_capture_job_status(
|
|
||||||
&conn, &job_uid_bg, "completed", None, None, None,
|
|
||||||
).ok();
|
|
||||||
} else {
|
|
||||||
// "not_a_tweet" or "scraper_failed" — surface as a job failure
|
|
||||||
// so the client sees a meaningful error message.
|
|
||||||
database::update_capture_job_status(
|
|
||||||
&conn, &job_uid_bg, "failed",
|
|
||||||
None, Some(&result.message), None,
|
|
||||||
).ok();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
database::update_capture_job_status(
|
|
||||||
&conn, &job_uid_bg, "failed",
|
|
||||||
None, Some(&format!("{e:#}")), None,
|
|
||||||
).ok();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
Ok((
|
|
||||||
StatusCode::ACCEPTED,
|
|
||||||
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
||||||
///
|
///
|
||||||
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
||||||
|
|
|
||||||
1
crates/archivr-server/static/assets/index-C0BIEdow.css
Normal file
1
crates/archivr-server/static/assets/index-C0BIEdow.css
Normal file
File diff suppressed because one or more lines are too long
40
crates/archivr-server/static/assets/index-D6iX-c2T.js
Normal file
40
crates/archivr-server/static/assets/index-D6iX-c2T.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>Archivr</title>
|
<title>Archivr</title>
|
||||||
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
|
<script type="module" crossorigin src="/assets/index-D6iX-c2T.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-C0BIEdow.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div id="root"></div>
|
<div id="root"></div>
|
||||||
|
|
|
||||||
|
|
@ -241,12 +241,10 @@ export default function App() {
|
||||||
])
|
])
|
||||||
}, [archiveId, searchQuery, tagFilter, loadEntries])
|
}, [archiveId, searchQuery, tagFilter, loadEntries])
|
||||||
|
|
||||||
const handleToast = useCallback((text, locator, type = 'error', headline = null) => {
|
const handleToast = useCallback((text, locator, type = 'error') => {
|
||||||
// Only suppress per-item ublock/cookie warnings (those carry a locator).
|
if (type === 'warning' && ublockWarningIgnored) return
|
||||||
// Batch summary warnings (locator = null) must always show.
|
|
||||||
if (type === 'warning' && ublockWarningIgnored && locator) return
|
|
||||||
const id = ++toastIdRef.current
|
const id = ++toastIdRef.current
|
||||||
setToasts(prev => [...prev, { id, text, locator, type, headline }])
|
setToasts(prev => [...prev, { id, text, locator, type }])
|
||||||
}, [ublockWarningIgnored])
|
}, [ublockWarningIgnored])
|
||||||
|
|
||||||
const handleDismissToast = useCallback((id) => {
|
const handleDismissToast = useCallback((id) => {
|
||||||
|
|
@ -256,7 +254,7 @@ export default function App() {
|
||||||
const handleIgnoreUblock = useCallback(() => {
|
const handleIgnoreUblock = useCallback(() => {
|
||||||
sessionStorage.setItem('ublockWarningIgnored', 'true')
|
sessionStorage.setItem('ublockWarningIgnored', 'true')
|
||||||
setUblockWarningIgnored(true)
|
setUblockWarningIgnored(true)
|
||||||
setToasts(prev => prev.filter(t => !(t.type === 'warning' && t.locator)))
|
setToasts(prev => prev.filter(t => t.type !== 'warning'))
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;
|
if (authState === 'loading') return <div className="auth-loading">Loading\u2026</div>;
|
||||||
|
|
|
||||||
|
|
@ -371,18 +371,6 @@ export async function deleteOrphanBlobs(archiveId) {
|
||||||
return res.json()
|
return res.json()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Re-archive ────────────────────────────────────────────────────────────────
|
|
||||||
export async function rearchiveEntry(archiveId, entryUid) {
|
|
||||||
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}/rearchive`, {
|
|
||||||
method: 'POST',
|
|
||||||
})
|
|
||||||
if (!res.ok) {
|
|
||||||
const body = await res.json().catch(() => ({}))
|
|
||||||
throw new Error(body.message || `rearchive failed: ${res.status}`)
|
|
||||||
}
|
|
||||||
return res.json() // { job_uid, status: 'pending' }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
// ── Cookie rules ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
export async function listCookieRules() {
|
export async function listCookieRules() {
|
||||||
|
|
|
||||||
|
|
@ -85,8 +85,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
const pollIntervals = useRef(new Map())
|
const pollIntervals = useRef(new Map())
|
||||||
// itemId → debounce timeoutId for probe calls
|
// itemId → debounce timeoutId for probe calls
|
||||||
const probeTimers = useRef(new Map())
|
const probeTimers = useRef(new Map())
|
||||||
// batchId → { total, archived, warnings, failed }; only populated for multi-URL submits
|
|
||||||
const batchRef = useRef(new Map())
|
|
||||||
// stable ref so probe callbacks always see the current archiveId
|
// stable ref so probe callbacks always see the current archiveId
|
||||||
const archiveIdRef = useRef(archiveId)
|
const archiveIdRef = useRef(archiveId)
|
||||||
useEffect(() => { archiveIdRef.current = archiveId }, [archiveId])
|
useEffect(() => { archiveIdRef.current = archiveId }, [archiveId])
|
||||||
|
|
@ -196,7 +194,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
}
|
}
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
function startPolling(itemId, jobUid, locator, aid, batchId = null) {
|
function startPolling(itemId, jobUid, locator, aid) {
|
||||||
if (pollIntervals.current.has(jobUid)) return // already polling
|
if (pollIntervals.current.has(jobUid)) return // already polling
|
||||||
const intervalId = setInterval(async () => {
|
const intervalId = setInterval(async () => {
|
||||||
try {
|
try {
|
||||||
|
|
@ -204,6 +202,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
if (updated.status === 'completed') {
|
if (updated.status === 'completed') {
|
||||||
clearInterval(pollIntervals.current.get(jobUid))
|
clearInterval(pollIntervals.current.get(jobUid))
|
||||||
pollIntervals.current.delete(jobUid)
|
pollIntervals.current.delete(jobUid)
|
||||||
|
// Show ✓ briefly then remove the row; if last row add a fresh one
|
||||||
setItems(prev => prev.map(it => it.id === itemId ? { ...it, status: 'completed' } : it))
|
setItems(prev => prev.map(it => it.id === itemId ? { ...it, status: 'completed' } : it))
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
setItems(prev => {
|
setItems(prev => {
|
||||||
|
|
@ -212,25 +211,13 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
})
|
})
|
||||||
}, 1400)
|
}, 1400)
|
||||||
onCapturedRef.current()
|
onCapturedRef.current()
|
||||||
|
// Warn if uBlock was requested but the extension wasn't available
|
||||||
try {
|
try {
|
||||||
const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null
|
const notes = updated.notes_json ? JSON.parse(updated.notes_json) : null
|
||||||
if (notes?.ublock_skipped || notes?.cookie_ext_skipped) {
|
if (notes?.ublock_skipped || notes?.cookie_ext_skipped) {
|
||||||
const both = notes.ublock_skipped && notes.cookie_ext_skipped
|
onToastRef.current(null, locator, 'warning')
|
||||||
const msg = both
|
|
||||||
? 'Captured without ad-blocking or cookie-consent extension. Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'
|
|
||||||
: notes.ublock_skipped
|
|
||||||
? 'Captured without ad-blocking. ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid.'
|
|
||||||
: 'Captured without cookie-consent extension. ARCHIVR_COOKIE_EXT is not set or the path is invalid.'
|
|
||||||
onToastRef.current(msg, locator, 'warning')
|
|
||||||
settleBatch(batchId, 'warning', locator)
|
|
||||||
} else {
|
|
||||||
if (!batchId) onToastRef.current(null, locator, 'success')
|
|
||||||
settleBatch(batchId, 'archived')
|
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {}
|
||||||
if (!batchId) onToastRef.current(null, locator, 'success')
|
|
||||||
settleBatch(batchId, 'archived')
|
|
||||||
}
|
|
||||||
} else if (updated.status === 'failed') {
|
} else if (updated.status === 'failed') {
|
||||||
clearInterval(pollIntervals.current.get(jobUid))
|
clearInterval(pollIntervals.current.get(jobUid))
|
||||||
pollIntervals.current.delete(jobUid)
|
pollIntervals.current.delete(jobUid)
|
||||||
|
|
@ -239,7 +226,6 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
it.id === itemId ? { ...it, status: 'failed', error: errText } : it
|
it.id === itemId ? { ...it, status: 'failed', error: errText } : it
|
||||||
))
|
))
|
||||||
onToastRef.current(errText, locator)
|
onToastRef.current(errText, locator)
|
||||||
settleBatch(batchId, 'failed', locator)
|
|
||||||
}
|
}
|
||||||
// 'pending' / 'running': keep polling
|
// 'pending' / 'running': keep polling
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|
@ -250,51 +236,14 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
it.id === itemId ? { ...it, status: 'failed', error: msg } : it
|
it.id === itemId ? { ...it, status: 'failed', error: msg } : it
|
||||||
))
|
))
|
||||||
onToastRef.current(msg, locator)
|
onToastRef.current(msg, locator)
|
||||||
settleBatch(batchId, 'failed', locator)
|
|
||||||
}
|
}
|
||||||
}, 500)
|
}, 500)
|
||||||
pollIntervals.current.set(jobUid, intervalId)
|
pollIntervals.current.set(jobUid, intervalId)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Increments the batch counter for the given outcome and emits a summary
|
async function submitItem(item) {
|
||||||
// toast once all jobs in the batch have settled.
|
|
||||||
// 'warning' counts as archived (succeeded with caveats); locator recorded for detail text.
|
|
||||||
function settleBatch(batchId, outcome, locator = null) {
|
|
||||||
if (!batchId) return
|
|
||||||
const batch = batchRef.current.get(batchId)
|
|
||||||
if (!batch) return
|
|
||||||
if (outcome === 'archived' || outcome === 'warning') {
|
|
||||||
batch.archived++
|
|
||||||
if (outcome === 'warning') { batch.warnings++; if (locator) batch.warningLocators.push(locator) }
|
|
||||||
} else {
|
|
||||||
batch.failed++
|
|
||||||
if (locator) batch.failedLocators.push(locator)
|
|
||||||
}
|
|
||||||
if (batch.archived + batch.failed < batch.total) return
|
|
||||||
// All settled — build headline + detail text, then emit summary and clean up.
|
|
||||||
batchRef.current.delete(batchId)
|
|
||||||
const { archived, warnings, failed, failedLocators, warningLocators } = batch
|
|
||||||
let headline
|
|
||||||
if (archived === 0) {
|
|
||||||
headline = `${failed} failed`
|
|
||||||
} else {
|
|
||||||
const archivedStr = warnings > 0
|
|
||||||
? `${archived} archived (${warnings} with warnings)`
|
|
||||||
: `${archived} archived`
|
|
||||||
headline = failed > 0 ? `${archivedStr}, ${failed} failed` : archivedStr
|
|
||||||
}
|
|
||||||
const type = archived === 0 ? 'error' : (failed > 0 || warnings > 0) ? 'warning' : 'success'
|
|
||||||
const parts = []
|
|
||||||
if (failedLocators.length > 0) parts.push(`Failed:\n${failedLocators.map(l => ` ${l}`).join('\n')}`)
|
|
||||||
if (warningLocators.length > 0) parts.push(`With warnings:\n${warningLocators.map(l => ` ${l}`).join('\n')}`)
|
|
||||||
const text = parts.length > 0 ? parts.join('\n') : null
|
|
||||||
onToastRef.current(text, null, type, headline)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async function submitItem(item, batchId = null) {
|
|
||||||
if (!item.locator.trim()) return
|
if (!item.locator.trim()) return
|
||||||
const aid = archiveId
|
const aid = archiveId // capture at submit time
|
||||||
const loc = item.locator.trim()
|
const loc = item.locator.trim()
|
||||||
const qual = item.quality || 'best'
|
const qual = item.quality || 'best'
|
||||||
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
|
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
|
||||||
|
|
@ -304,27 +253,17 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
|
||||||
setItems(prev => prev.map(it =>
|
setItems(prev => prev.map(it =>
|
||||||
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
|
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
|
||||||
))
|
))
|
||||||
startPolling(item.id, job.job_uid, loc, aid, batchId)
|
startPolling(item.id, job.job_uid, loc, aid)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const msg = e.message || 'Submission failed.'
|
const msg = e.message || 'Submission failed.'
|
||||||
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'failed', error: msg } : it))
|
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'failed', error: msg } : it))
|
||||||
onToastRef.current(msg, loc)
|
onToastRef.current(msg, loc)
|
||||||
settleBatch(batchId, 'failed', loc)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function handleArchive() {
|
function handleArchive() {
|
||||||
const toSubmit = items.filter(it => it.status === 'idle' && it.locator.trim())
|
const toSubmit = items.filter(it => it.status === 'idle' && it.locator.trim())
|
||||||
if (toSubmit.length === 0) return
|
toSubmit.forEach(it => submitItem(it))
|
||||||
const batchId = toSubmit.length > 1
|
|
||||||
? (crypto.randomUUID?.() ?? `batch-${Date.now()}`)
|
|
||||||
: null
|
|
||||||
if (batchId) {
|
|
||||||
batchRef.current.set(batchId, { total: toSubmit.length, archived: 0, warnings: 0, failed: 0, failedLocators: [], warningLocators: [] })
|
|
||||||
}
|
|
||||||
toSubmit.forEach(it => submitItem(it, batchId))
|
|
||||||
dialogRef.current?.close()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function addRow() {
|
function addRow() {
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { useState, useEffect, useRef } from 'react'
|
import { useState, useEffect, useRef } from 'react'
|
||||||
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry, rearchiveEntry, pollCaptureJob } from '../api'
|
import { fetchEntryDetail, fetchEntryTags, assignTag, removeTag, listEntryCollections, updateEntryTitle, deleteEntry } from '../api'
|
||||||
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
|
import { formatTimestamp, formatBytes, valueText, sourceIconSvg, displayPath } from '../utils'
|
||||||
|
|
||||||
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
|
const VIS_LABEL = { 0: 'Private', 1: 'Public', 2: 'Users only', 3: 'Public' }
|
||||||
|
|
@ -21,14 +21,8 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
const titleCancelRef = useRef(false)
|
const titleCancelRef = useRef(false)
|
||||||
const [editingTitle, setEditingTitle] = useState(false)
|
const [editingTitle, setEditingTitle] = useState(false)
|
||||||
const [titleDraft, setTitleDraft] = useState('')
|
const [titleDraft, setTitleDraft] = useState('')
|
||||||
const [rearchiveState, setRearchiveState] = useState('idle') // 'idle' | 'running' | 'done' | 'error'
|
|
||||||
const [rearchiveError, setRearchiveError] = useState('')
|
|
||||||
const rearchivePollRef = useRef(null)
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (rearchivePollRef.current) { clearInterval(rearchivePollRef.current); rearchivePollRef.current = null }
|
|
||||||
setRearchiveState('idle')
|
|
||||||
setRearchiveError('')
|
|
||||||
if (!selectedEntry || !archiveId) {
|
if (!selectedEntry || !archiveId) {
|
||||||
setDetail(null)
|
setDetail(null)
|
||||||
setTags([])
|
setTags([])
|
||||||
|
|
@ -53,12 +47,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
}).catch(() => {})
|
}).catch(() => {})
|
||||||
}, [selectedEntry, archiveId])
|
}, [selectedEntry, archiveId])
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
return () => {
|
|
||||||
clearInterval(rearchivePollRef.current)
|
|
||||||
}
|
|
||||||
}, [])
|
|
||||||
|
|
||||||
async function handleTitleSave() {
|
async function handleTitleSave() {
|
||||||
const newTitle = titleDraft.trim() || null
|
const newTitle = titleDraft.trim() || null
|
||||||
try {
|
try {
|
||||||
|
|
@ -109,46 +97,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleRearchive() {
|
|
||||||
if (!selectedEntry || !archiveId || rearchiveState === 'running') return
|
|
||||||
setRearchiveState('running')
|
|
||||||
setRearchiveError('')
|
|
||||||
try {
|
|
||||||
const { job_uid } = await rearchiveEntry(archiveId, selectedEntry.entry_uid)
|
|
||||||
// Poll for job completion at 500ms — same interval as CaptureDialog
|
|
||||||
rearchivePollRef.current = setInterval(async () => {
|
|
||||||
try {
|
|
||||||
const job = await pollCaptureJob(archiveId, job_uid)
|
|
||||||
if (job.status === 'completed') {
|
|
||||||
clearInterval(rearchivePollRef.current)
|
|
||||||
rearchivePollRef.current = null
|
|
||||||
setRearchiveState('done')
|
|
||||||
// Refresh entry detail so artifact list updates
|
|
||||||
const [det, tgs] = await Promise.all([
|
|
||||||
fetchEntryDetail(archiveId, selectedEntry.entry_uid),
|
|
||||||
fetchEntryTags(archiveId, selectedEntry.entry_uid),
|
|
||||||
])
|
|
||||||
setDetail(det)
|
|
||||||
setTags(tgs)
|
|
||||||
} else if (job.status === 'failed') {
|
|
||||||
clearInterval(rearchivePollRef.current)
|
|
||||||
rearchivePollRef.current = null
|
|
||||||
setRearchiveState('error')
|
|
||||||
setRearchiveError(job.error_text || 'Re-archive failed.')
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
clearInterval(rearchivePollRef.current)
|
|
||||||
rearchivePollRef.current = null
|
|
||||||
setRearchiveState('error')
|
|
||||||
setRearchiveError('Network error while polling.')
|
|
||||||
}
|
|
||||||
}, 500)
|
|
||||||
} catch (e) {
|
|
||||||
setRearchiveState('error')
|
|
||||||
setRearchiveError(e.message || 'Failed to start re-archive.')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const metaRows = detail ? [
|
const metaRows = detail ? [
|
||||||
['Added', formatTimestamp(detail.summary.archived_at)],
|
['Added', formatTimestamp(detail.summary.archived_at)],
|
||||||
['Source', detail.summary.source_kind],
|
['Source', detail.summary.source_kind],
|
||||||
|
|
@ -296,25 +244,6 @@ export default function ContextRail({ archiveId, selectedEntry, onTagFilterSet,
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{detail && (detail.summary.entity_kind === 'tweet' || detail.summary.entity_kind === 'tweet_thread') && (
|
|
||||||
<div className="rail-section">
|
|
||||||
<div className="rail-section-heading">Actions</div>
|
|
||||||
<button
|
|
||||||
className="rail-rearchive-btn"
|
|
||||||
onClick={handleRearchive}
|
|
||||||
disabled={rearchiveState === 'running'}
|
|
||||||
>
|
|
||||||
{rearchiveState === 'running' ? 'Re-archiving\u2026' : 'Re-archive'}
|
|
||||||
</button>
|
|
||||||
{rearchiveState === 'done' && (
|
|
||||||
<p className="form-msg form-msg--ok" style={{ marginTop: '6px' }}>Re-archived successfully.</p>
|
|
||||||
)}
|
|
||||||
{rearchiveState === 'error' && (
|
|
||||||
<p className="form-msg form-msg--err" style={{ marginTop: '6px' }}>{rearchiveError}</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
<div className="rail-delete-zone">
|
<div className="rail-delete-zone">
|
||||||
<button className="rail-delete-btn" onClick={handleDeleteEntry}>
|
<button className="rail-delete-btn" onClick={handleDeleteEntry}>
|
||||||
Delete entry
|
Delete entry
|
||||||
|
|
|
||||||
|
|
@ -13,27 +13,9 @@ export default function ToastStack({ toasts, onDismiss, onIgnoreUblock }) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// For URLs: show hostname/…/tail (drops protocol + middle path, keeps domain + identifier).
|
|
||||||
// For shorthands (yt:, x:, etc.): keep the tail since it's the whole identifier.
|
|
||||||
// CSS text-overflow:ellipsis on .toast-locator handles any remaining overflow.
|
|
||||||
function shortLocator(locator) {
|
|
||||||
if (!locator) return null
|
|
||||||
if (locator.length <= 52) return locator
|
|
||||||
try {
|
|
||||||
const { hostname, pathname, search } = new URL(locator)
|
|
||||||
const segments = pathname.split('/').filter(Boolean)
|
|
||||||
const tail = (segments[segments.length - 1] ?? '') + search
|
|
||||||
const candidate = tail ? `${hostname}/\u2026/${tail}` : hostname
|
|
||||||
return candidate.length <= 56 ? candidate : candidate.slice(0, 53) + '\u2026'
|
|
||||||
} catch {
|
|
||||||
return '\u2026' + locator.slice(-51)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||||
const [expanded, setExpanded] = useState(false)
|
const [expanded, setExpanded] = useState(false)
|
||||||
const isWarning = toast.type === 'warning'
|
const isWarning = toast.type === 'warning'
|
||||||
const isSuccess = toast.type === 'success'
|
|
||||||
|
|
||||||
// Auto-dismiss after TTL; paused while detail is expanded
|
// Auto-dismiss after TTL; paused while detail is expanded
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|
@ -42,24 +24,9 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||||
return () => clearTimeout(timer)
|
return () => clearTimeout(timer)
|
||||||
}, [expanded, toast.id, onDismiss])
|
}, [expanded, toast.id, onDismiss])
|
||||||
|
|
||||||
const short = shortLocator(toast.locator)
|
const short = toast.locator
|
||||||
|
? (toast.locator.length > 48 ? toast.locator.slice(0, 45) + '\u2026' : toast.locator)
|
||||||
if (isSuccess) {
|
: null
|
||||||
return (
|
|
||||||
<div className="toast toast--success" role="alert" aria-atomic="true">
|
|
||||||
<div className="toast-top">
|
|
||||||
<span className="toast-icon" aria-hidden="true">✓</span>
|
|
||||||
<div className="toast-body">
|
|
||||||
<span className="toast-headline">{toast.headline || 'Archived'}</span>
|
|
||||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
|
||||||
</div>
|
|
||||||
<div className="toast-btns">
|
|
||||||
<button type="button" className="toast-dismiss" onClick={() => onDismiss(toast.id)} aria-label="Dismiss">×</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isWarning) {
|
if (isWarning) {
|
||||||
return (
|
return (
|
||||||
|
|
@ -67,29 +34,25 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||||
<div className="toast-top">
|
<div className="toast-top">
|
||||||
<span className="toast-icon" aria-hidden="true">⚠</span>
|
<span className="toast-icon" aria-hidden="true">⚠</span>
|
||||||
<div className="toast-body">
|
<div className="toast-body">
|
||||||
<span className="toast-headline">{toast.headline || 'Archived with warnings'}</span>
|
<span className="toast-headline">Ad-blocking unavailable</span>
|
||||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
{short && <span className="toast-locator">{short}</span>}
|
||||||
</div>
|
</div>
|
||||||
<div className="toast-btns">
|
<div className="toast-btns">
|
||||||
{toast.text && (
|
<button
|
||||||
<button
|
type="button"
|
||||||
type="button"
|
className="toast-view-btn"
|
||||||
className="toast-view-btn"
|
onClick={() => setExpanded(v => !v)}
|
||||||
onClick={() => setExpanded(v => !v)}
|
aria-expanded={expanded}
|
||||||
aria-expanded={expanded}
|
>
|
||||||
>
|
{expanded ? 'Hide' : 'Details'}
|
||||||
{expanded ? 'Hide' : 'Details'}
|
</button>
|
||||||
</button>
|
<button
|
||||||
)}
|
type="button"
|
||||||
{toast.locator && (
|
className="toast-view-btn toast-ignore-btn"
|
||||||
<button
|
onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }}
|
||||||
type="button"
|
>
|
||||||
className="toast-view-btn toast-ignore-btn"
|
Ignore
|
||||||
onClick={() => { onIgnoreUblock?.(); onDismiss(toast.id) }}
|
</button>
|
||||||
>
|
|
||||||
Ignore
|
|
||||||
</button>
|
|
||||||
)}
|
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
className="toast-dismiss"
|
className="toast-dismiss"
|
||||||
|
|
@ -102,7 +65,7 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||||
</div>
|
</div>
|
||||||
{expanded && (
|
{expanded && (
|
||||||
<p className="toast-warning-detail">
|
<p className="toast-warning-detail">
|
||||||
{toast.text || 'The page was captured but one or more browser extensions were unavailable (ad-blocking or cookie-consent). Check ARCHIVR_UBLOCK_EXT and ARCHIVR_COOKIE_EXT in your server config.'}
|
{toast.text || 'ARCHIVR_UBLOCK=true but ARCHIVR_UBLOCK_EXT is not set or the path is invalid. The page was captured without ad-blocking. Set ARCHIVR_UBLOCK_EXT to the unpacked uBlock Origin Lite extension directory to enable ad-blocking, or set ARCHIVR_UBLOCK=false to silence this warning.'}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -114,8 +77,8 @@ function Toast({ toast, onDismiss, onIgnoreUblock }) {
|
||||||
<div className="toast-top">
|
<div className="toast-top">
|
||||||
<span className="toast-icon" aria-hidden="true">✕</span>
|
<span className="toast-icon" aria-hidden="true">✕</span>
|
||||||
<div className="toast-body">
|
<div className="toast-body">
|
||||||
<span className="toast-headline">{toast.headline || 'Capture failed'}</span>
|
<span className="toast-headline">Capture failed</span>
|
||||||
{short && <span className="toast-locator" title={toast.locator}>{short}</span>}
|
{short && <span className="toast-locator">{short}</span>}
|
||||||
</div>
|
</div>
|
||||||
<div className="toast-btns">
|
<div className="toast-btns">
|
||||||
{toast.text && (
|
{toast.text && (
|
||||||
|
|
|
||||||
|
|
@ -552,24 +552,6 @@ select {
|
||||||
border-color: var(--accent);
|
border-color: var(--accent);
|
||||||
}
|
}
|
||||||
|
|
||||||
.rail-rearchive-btn {
|
|
||||||
width: 100%;
|
|
||||||
padding: 6px 12px;
|
|
||||||
background: var(--surface-2, #2a2a2a);
|
|
||||||
color: var(--text, #e0e0e0);
|
|
||||||
border: 1px solid var(--border, #444);
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
font-size: 13px;
|
|
||||||
}
|
|
||||||
.rail-rearchive-btn:hover:not(:disabled) {
|
|
||||||
background: var(--surface-3, #333);
|
|
||||||
}
|
|
||||||
.rail-rearchive-btn:disabled {
|
|
||||||
opacity: 0.6;
|
|
||||||
cursor: not-allowed;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ── Capture dialog ──────────────────────────────────────────────────────── */
|
/* ── Capture dialog ──────────────────────────────────────────────────────── */
|
||||||
.capture-dialog {
|
.capture-dialog {
|
||||||
border: 1px solid var(--line);
|
border: 1px solid var(--line);
|
||||||
|
|
@ -887,7 +869,6 @@ select {
|
||||||
}
|
}
|
||||||
.toast--error { border-left: 3px solid var(--accent); }
|
.toast--error { border-left: 3px solid var(--accent); }
|
||||||
.toast--warning { border-left: 3px solid #e8a000; }
|
.toast--warning { border-left: 3px solid #e8a000; }
|
||||||
.toast--success { border-left: 3px solid #22a855; }
|
|
||||||
|
|
||||||
.toast-top {
|
.toast-top {
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|
@ -902,8 +883,6 @@ select {
|
||||||
color: var(--accent);
|
color: var(--accent);
|
||||||
margin-top: 2px;
|
margin-top: 2px;
|
||||||
}
|
}
|
||||||
.toast--success .toast-icon { color: #22a855; }
|
|
||||||
.toast--warning .toast-icon { color: #e8a000; }
|
|
||||||
.toast-body {
|
.toast-body {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue