1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

feat(tweets): add re-archive button and fix thread-tweet orphan cleanup (#23)

Orphan cleanup bug: archiving x🧵A downloaded D/C/B/A JSONs and
media, but only registered artifacts for A. D/C/B files had no
entry_artifacts rows and were deleted as orphans.

Fix (staged scraper output, precise touched set):
- tweets::archive() stages all scraper output in temp/{ts}/tweet_stage/,
  validates, then renames JSONs to raw_tweets/. Return type changed from
  Result<bool> to Result<Vec<String>> (store-relative relpaths of every
  produced tweet JSON, i.e. the exact touched set).
- tweets::rearchive() (new): same staged approach but always runs the
  scraper. On scraper failure (tweet deleted/private), errors before
  touching raw_tweets/ so existing data is preserved.
- register_tweet_artifacts() (new private helper in capture.rs): registers
  every JSON in the touched set as a raw_tweet_json artifact, parses each
  for media blobs, registers those too. JSON read failure is a hard error
  with context, not a silent skip.
- record_tweet_entry() now accepts tweet_json_relpaths: &[String] and
  delegates artifact registration to register_tweet_artifacts().
- perform_capture() passes the returned vec from tweets::archive().

Re-archive feature:
- capture::perform_rearchive(): looks up entry by uid, validates
  tweet/tweet_thread, runs tweets::rearchive(), atomically swaps
  entry_artifacts in a DB transaction. archived_at, title, tags,
  collections are untouched.
- database: add get_entry_for_rearchive() and delete_entry_artifacts().
- POST /api/archives/:id/entries/:uid/rearchive: requires ROLE_USER,
  creates capture job, returns 202 + job_uid, runs perform_rearchive in
  spawn_blocking.
- Frontend: re-archive button in ContextRail for tweet/tweet_thread
  entries; polls job at 500ms; refreshes entry detail on success; shows
  error text on failure. Poll interval cleared before early-return on
  entry deselect to prevent stale updates.
This commit is contained in:
TheGeneralist 2026-07-11 16:14:58 +02:00 committed by GitHub
parent 03390362c5
commit 2779afee2d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 528 additions and 174 deletions

View file

@ -221,6 +221,10 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
get(serve_artifact),
)
.route(
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
post(rearchive_handler),
)
.route(
"/api/archives/:archive_id/entries/:entry_uid/favicon",
get(serve_entry_favicon),
@ -832,6 +836,83 @@ async fn get_capture_job_handler(
.ok_or_else(|| ApiError::not_found("capture job not found"))
}
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
///
/// Re-archives an existing tweet or tweet_thread entry in-place:
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
///
/// Returns 202 immediately with a job_uid the client should poll via
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
async fn rearchive_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path((archive_id, entry_uid)): Path<(String, String)>,
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
auth_user.require_role(ROLE_USER)?;
let mounted = mounted_archive(&state, &archive_id)?;
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
.map_err(ApiError::from)?;
// Create a capture job record so the client can poll for completion.
let conn = database::open_or_initialize(&mounted.archive_path)?;
let job_uid = database::create_capture_job(&conn, &archive_id)?;
drop(conn);
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
Err(_) => vec![],
};
let capture_config = capture::CaptureConfig {
cookie_rules,
ublock_enabled: None,
cookie_ext_enabled: None,
reader_mode: false,
};
let job_uid_bg = job_uid.clone();
let archive_path = mounted.archive_path.clone();
tokio::task::spawn_blocking(move || {
let conn = match database::open_or_initialize(&archive_path) {
Ok(c) => c,
Err(e) => {
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
return;
}
};
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
Ok(result) => {
if result.status == "completed" {
database::update_capture_job_status(
&conn, &job_uid_bg, "completed", None, None, None,
).ok();
} else {
// "not_a_tweet" or "scraper_failed" — surface as a job failure
// so the client sees a meaningful error message.
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&result.message), None,
).ok();
}
}
Err(e) => {
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&format!("{e:#}")), None,
).ok();
}
}
});
Ok((
StatusCode::ACCEPTED,
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
))
}
/// `GET /api/archives/:id/captures/probe?locator=<url>`
///
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,8 +4,8 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title>
<script type="module" crossorigin src="/assets/index-BbGz9qlG.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-D0Nzcia0.css">
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
</head>
<body>
<div id="root"></div>