mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat(tweets): add re-archive button and fix thread-tweet orphan cleanup (#23)
Orphan cleanup bug: archiving x🧵A downloaded D/C/B/A JSONs and
media, but only registered artifacts for A. D/C/B files had no
entry_artifacts rows and were deleted as orphans.
Fix (staged scraper output, precise touched set):
- tweets::archive() stages all scraper output in temp/{ts}/tweet_stage/,
validates, then renames JSONs to raw_tweets/. Return type changed from
Result<bool> to Result<Vec<String>> (store-relative relpaths of every
produced tweet JSON, i.e. the exact touched set).
- tweets::rearchive() (new): same staged approach but always runs the
scraper. On scraper failure (tweet deleted/private), errors before
touching raw_tweets/ so existing data is preserved.
- register_tweet_artifacts() (new private helper in capture.rs): registers
every JSON in the touched set as a raw_tweet_json artifact, parses each
for media blobs, registers those too. JSON read failure is a hard error
with context, not a silent skip.
- record_tweet_entry() now accepts tweet_json_relpaths: &[String] and
delegates artifact registration to register_tweet_artifacts().
- perform_capture() passes the returned vec from tweets::archive().
Re-archive feature:
- capture::perform_rearchive(): looks up entry by uid, validates
tweet/tweet_thread, runs tweets::rearchive(), atomically swaps
entry_artifacts in a DB transaction. archived_at, title, tags,
collections are untouched.
- database: add get_entry_for_rearchive() and delete_entry_artifacts().
- POST /api/archives/:id/entries/:uid/rearchive: requires ROLE_USER,
creates capture job, returns 202 + job_uid, runs perform_rearchive in
spawn_blocking.
- Frontend: re-archive button in ContextRail for tweet/tweet_thread
entries; polls job at 500ms; refreshes entry detail on success; shows
error text on failure. Poll interval cleared before early-return on
entry deselect to prevent stale updates.
This commit is contained in:
parent
03390362c5
commit
2779afee2d
12 changed files with 528 additions and 174 deletions
|
|
@ -221,6 +221,10 @@ pub fn app_with_state(state: AppState) -> Router {
|
|||
"/api/archives/:archive_id/entries/:entry_uid/artifacts/:artifact_index",
|
||||
get(serve_artifact),
|
||||
)
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid/rearchive",
|
||||
post(rearchive_handler),
|
||||
)
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid/favicon",
|
||||
get(serve_entry_favicon),
|
||||
|
|
@ -832,6 +836,83 @@ async fn get_capture_job_handler(
|
|||
.ok_or_else(|| ApiError::not_found("capture job not found"))
|
||||
}
|
||||
|
||||
/// POST /api/archives/:archive_id/entries/:entry_uid/rearchive
|
||||
///
|
||||
/// Re-archives an existing tweet or tweet_thread entry in-place:
|
||||
/// - Stages scraper output in a temp dir (existing data safe if scraper fails)
|
||||
/// - On success: atomically replaces entry_artifacts and refreshes cached_bytes
|
||||
/// - On failure (tweet deleted/private): job is marked failed; existing data preserved
|
||||
///
|
||||
/// Returns 202 immediately with a job_uid the client should poll via
|
||||
/// GET /api/archives/:archive_id/capture_jobs/:job_uid.
|
||||
async fn rearchive_handler(
|
||||
State(state): State<AppState>,
|
||||
auth_user: AuthUser,
|
||||
Path((archive_id, entry_uid)): Path<(String, String)>,
|
||||
) -> Result<(StatusCode, Json<serde_json::Value>), ApiError> {
|
||||
auth_user.require_role(ROLE_USER)?;
|
||||
let mounted = mounted_archive(&state, &archive_id)?;
|
||||
let archive_paths = archive::read_archive_paths(&mounted.archive_path)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// Create a capture job record so the client can poll for completion.
|
||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||
let job_uid = database::create_capture_job(&conn, &archive_id)?;
|
||||
drop(conn);
|
||||
|
||||
// Load cookie rules from the auth DB (needed for Twitter credentials resolution).
|
||||
let cookie_rules = match database::open_auth_db(&state.auth_db_path) {
|
||||
Ok(conn) => database::list_cookie_rules(&conn).unwrap_or_default(),
|
||||
Err(_) => vec![],
|
||||
};
|
||||
let capture_config = capture::CaptureConfig {
|
||||
cookie_rules,
|
||||
ublock_enabled: None,
|
||||
cookie_ext_enabled: None,
|
||||
reader_mode: false,
|
||||
};
|
||||
|
||||
let job_uid_bg = job_uid.clone();
|
||||
let archive_path = mounted.archive_path.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let conn = match database::open_or_initialize(&archive_path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!("warn: rearchive job {job_uid_bg}: db open failed: {e:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
database::update_capture_job_status(&conn, &job_uid_bg, "running", None, None, None).ok();
|
||||
match capture::perform_rearchive(&archive_paths, &entry_uid, &capture_config) {
|
||||
Ok(result) => {
|
||||
if result.status == "completed" {
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "completed", None, None, None,
|
||||
).ok();
|
||||
} else {
|
||||
// "not_a_tweet" or "scraper_failed" — surface as a job failure
|
||||
// so the client sees a meaningful error message.
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "failed",
|
||||
None, Some(&result.message), None,
|
||||
).ok();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
database::update_capture_job_status(
|
||||
&conn, &job_uid_bg, "failed",
|
||||
None, Some(&format!("{e:#}")), None,
|
||||
).ok();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok((
|
||||
StatusCode::ACCEPTED,
|
||||
Json(serde_json::json!({ "job_uid": job_uid, "status": "pending" })),
|
||||
))
|
||||
}
|
||||
|
||||
/// `GET /api/archives/:id/captures/probe?locator=<url>`
|
||||
///
|
||||
/// Runs `yt-dlp --dump-json` (behind `spawn_blocking`) and returns the video
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
45
crates/archivr-server/static/assets/index-DiHAMIVH.js
Normal file
45
crates/archivr-server/static/assets/index-DiHAMIVH.js
Normal file
File diff suppressed because one or more lines are too long
1
crates/archivr-server/static/assets/index-DwI288Sc.css
Normal file
1
crates/archivr-server/static/assets/index-DwI288Sc.css
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@
|
|||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Archivr</title>
|
||||
<script type="module" crossorigin src="/assets/index-BbGz9qlG.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-D0Nzcia0.css">
|
||||
<script type="module" crossorigin src="/assets/index-DiHAMIVH.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DwI288Sc.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue