1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 03:05:32 +02:00

feat(tweets): add re-archive button and fix thread-tweet orphan cleanup (#23)

Orphan cleanup bug: archiving x🧵A downloaded D/C/B/A JSONs and
media, but only registered artifacts for A. D/C/B files had no
entry_artifacts rows and were deleted as orphans.

Fix (staged scraper output, precise touched set):
- tweets::archive() stages all scraper output in temp/{ts}/tweet_stage/,
  validates, then renames JSONs to raw_tweets/. Return type changed from
  Result<bool> to Result<Vec<String>> (store-relative relpaths of every
  produced tweet JSON, i.e. the exact touched set).
- tweets::rearchive() (new): same staged approach but always runs the
  scraper. On scraper failure (tweet deleted/private), errors before
  touching raw_tweets/ so existing data is preserved.
- register_tweet_artifacts() (new private helper in capture.rs): registers
  every JSON in the touched set as a raw_tweet_json artifact, parses each
  for media blobs, registers those too. JSON read failure is a hard error
  with context, not a silent skip.
- record_tweet_entry() now accepts tweet_json_relpaths: &[String] and
  delegates artifact registration to register_tweet_artifacts().
- perform_capture() passes the returned vec from tweets::archive().

Re-archive feature:
- capture::perform_rearchive(): looks up entry by uid, validates
  tweet/tweet_thread, runs tweets::rearchive(), atomically swaps
  entry_artifacts in a DB transaction. archived_at, title, tags,
  collections are untouched.
- database: add get_entry_for_rearchive() and delete_entry_artifacts().
- POST /api/archives/:id/entries/:uid/rearchive: requires ROLE_USER,
  creates capture job, returns 202 + job_uid, runs perform_rearchive in
  spawn_blocking.
- Frontend: re-archive button in ContextRail for tweet/tweet_thread
  entries; polls job at 500ms; refreshes entry detail on success; shows
  error text on failure. Poll interval cleared before early-return on
  entry deselect to prevent stale updates.
This commit is contained in:
TheGeneralist 2026-07-11 16:14:58 +02:00 committed by GitHub
parent 03390362c5
commit 2779afee2d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 528 additions and 174 deletions

View file

@ -221,6 +221,10 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
get(serve_artifact),
)
.route(
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
post(rearchive_handler),
)
.route(
"/api/archives/:archive_id/entries/:entry_uid/favicon",
get(serve_entry_favicon),
@ -832,6 +836,83 @@ async fn get_capture_job_handler(
.ok_or_else(|| ApiError::not_found("capture job not found"))
}
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
///
/// Re-archives an existing tweet or tweet_thread entry in-place:
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
///
/// Returns 202 immediately with a job_uid the client should poll via
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
async fn rearchive_handler(
State(state): State<AppState>,
auth_user: AuthUser,
Path((archive_id, entry_uid)): Path<(String, String)>,
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
auth_user.require_role(ROLE_USER)?;
let mounted = mounted_archive(&state, &archive_id)?;
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
.map_err(ApiError::from)?;
// Create a capture job record so the client can poll for completion.
let conn = database::open_or_initialize(&mounted.archive_path)?;
let job_uid = database::create_capture_job(&conn, &archive_id)?;
drop(conn);
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
Err(_) => vec![],
};
let capture_config = capture::CaptureConfig {
cookie_rules,
ublock_enabled: None,
cookie_ext_enabled: None,
reader_mode: false,
};
let job_uid_bg = job_uid.clone();
let archive_path = mounted.archive_path.clone();
tokio::task::spawn_blocking(move || {
let conn = match database::open_or_initialize(&archive_path) {
Ok(c) => c,
Err(e) => {
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
return;
}
};
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
Ok(result) => {
if result.status == "completed" {
database::update_capture_job_status(
&conn, &job_uid_bg, "completed", None, None, None,
).ok();
} else {
// "not_a_tweet" or "scraper_failed" — surface as a job failure
// so the client sees a meaningful error message.
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&result.message), None,
).ok();
}
}
Err(e) => {
database::update_capture_job_status(
&conn, &job_uid_bg, "failed",
None, Some(&format!("{e:#}")), None,
).ok();
}
}
});
Ok((
StatusCode::ACCEPTED,
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
))
}
/// `GET /api/archives/:id/captures/probe?locator=<url>`
///
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video