thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00
Code Issues Activity

feat(tweets): add re-archive button and fix thread-tweet orphan cleanup (#23)

Browse source 
Orphan cleanup bug: archiving x🧵A downloaded D/C/B/A JSONs and
media, but only registered artifacts for A. D/C/B files had no
entry_artifacts rows and were deleted as orphans.

Fix (staged scraper output, precise touched set):
- tweets::archive() stages all scraper output in temp/{ts}/tweet_stage/,
  validates, then renames JSONs to raw_tweets/. Return type changed from
  Result<bool> to Result<Vec<String>> (store-relative relpaths of every
  produced tweet JSON, i.e. the exact touched set).
- tweets::rearchive() (new): same staged approach but always runs the
  scraper. On scraper failure (tweet deleted/private), errors before
  touching raw_tweets/ so existing data is preserved.
- register_tweet_artifacts() (new private helper in capture.rs): registers
  every JSON in the touched set as a raw_tweet_json artifact, parses each
  for media blobs, registers those too. JSON read failure is a hard error
  with context, not a silent skip.
- record_tweet_entry() now accepts tweet_json_relpaths: &[String] and
  delegates artifact registration to register_tweet_artifacts().
- perform_capture() passes the returned vec from tweets::archive().

Re-archive feature:
- capture::perform_rearchive(): looks up entry by uid, validates
  tweet/tweet_thread, runs tweets::rearchive(), atomically swaps
  entry_artifacts in a DB transaction. archived_at, title, tags,
  collections are untouched.
- database: add get_entry_for_rearchive() and delete_entry_artifacts().
- POST /api/archives/:id/entries/:uid/rearchive: requires ROLE_USER,
  creates capture job, returns 202 + job_uid, runs perform_rearchive in
  spawn_blocking.
- Frontend: re-archive button in ContextRail for tweet/tweet_thread
  entries; polls job at 500ms; refreshes entry detail on success; shows
  error text on failure. Poll interval cleared before early-return on
  entry deselect to prevent stale updates.
This commit is contained in:
TheGeneralist 2026-07-11 16:14:58 +02:00 • committed by GitHub
parent 03390362c5
commit 2779afee2d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 528 additions and 174 deletions
Download patch file Download diff file Expand all files Collapse all files

186
crates/archivr-core/src/capture.rs
View file

@ -742,6 +742,52 @@ fn tweet_metadata_from_json(json_str: &str) -> PlatformMetadata {
}
}
/// Registers all tweet JSON files and their media blobs as entry_artifacts.
/// Called by both record_tweet_entry (new captures) and perform_rearchive (re-captures).
/// `tweet_json_relpaths`: store-relative paths like `"raw_tweets/tweet-123.json"`.
fn register_tweet_artifacts(
conn: &rusqlite::Connection,
store_path: &Path,
entry_id: i64,
tweet_json_relpaths: &[String],
) -> Result<()> {
for relpath in tweet_json_relpaths {
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id,
artifact_role: "raw_tweet_json".to_string(),
storage_area: "raw_tweets".to_string(),
relpath: relpath.clone(),
blob_id: None,
logical_path: None,
metadata_json: None,
},
)?;
let json_path = store_path.join(relpath);
let json_str = fs::read_to_string(&json_path)
.with_context(|| format!("failed to read tweet JSON for artifact registration: {}", json_path.display()))?;
for (role, raw_relpath) in tweet_raw_artifacts(&json_str)? {
let raw_path = PathBuf::from(&raw_relpath);
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
let blob_id = database::upsert_blob(conn, &blob)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id,
artifact_role: role,
storage_area: "raw".to_string(),
relpath: raw_relpath,
blob_id: Some(blob_id),
logical_path: None,
metadata_json: None,
},
)?;
}
}
Ok(())
}
fn record_tweet_entry(
conn: &rusqlite::Connection,
store_path: &Path,
@ -751,6 +797,7 @@ fn record_tweet_entry(
requested_locator: &str,
source: Source,
tweet_id: &str,
tweet_json_relpaths: &[String],
) -> Result<database::ArchivedEntry> {
debug_assert!(run.run_uid.starts_with("run_"));
debug_assert!(item.item_uid.starts_with("item_"));
@ -764,7 +811,7 @@ fn record_tweet_entry(
Some(&canonical_locator),
&canonical_locator,
)?;
// Read tweet JSON early to extract title before entry creation
// Read the primary tweet JSON to extract title before entry creation.
let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json"));
let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?;
let tweet_meta = tweet_metadata_from_json(&tweet_json);
@ -794,36 +841,8 @@ fn record_tweet_entry(
)?;
create_structured_root(store_path, &entry)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id: entry.id,
artifact_role: "raw_tweet_json".to_string(),
storage_area: "raw_tweets".to_string(),
relpath: path_to_store_string(&tweet_json_relpath),
blob_id: None,
logical_path: None,
metadata_json: None,
},
)?;
for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? {
let raw_path = PathBuf::from(&raw_relpath);
let blob = blob_record_for_raw_relpath(store_path, &raw_path)?;
let blob_id = database::upsert_blob(conn, &blob)?;
database::add_entry_artifact(
conn,
&database::NewArtifact {
entry_id: entry.id,
artifact_role: role,
storage_area: "raw".to_string(),
relpath: raw_relpath,
blob_id: Some(blob_id),
logical_path: None,
metadata_json: None,
},
)?;
}
// Register all tweet JSONs and their blobs.
register_tweet_artifacts(conn, store_path, entry.id, tweet_json_relpaths)?;
database::complete_archive_run_item(conn, item.id, entry.id)?;
Ok(entry)
@ -1184,7 +1203,7 @@ pub fn perform_capture(
&timestamp,
&tweet_cookies,
) {
Ok(_) => {
Ok(tweet_json_relpaths) => {
let tweet_entry = record_tweet_entry(
&conn,
store_path,
@ -1194,6 +1213,7 @@ pub fn perform_capture(
locator,
source,
&tweet_id,
&tweet_json_relpaths,
)?;
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
database::finish_archive_run(&conn, run.id)?;
@ -1362,6 +1382,107 @@ pub fn perform_capture(
})
}
/// Result of a tweet re-archive operation.
#[derive(Debug, serde::Serialize)]
pub struct RearchiveResult {
/// One of: "completed", "not_a_tweet", "scraper_failed".
pub status: String,
/// Human-readable detail (empty string for "completed").
pub message: String,
}
/// Re-archives an existing tweet or tweet_thread entry in-place.
///
/// - Looks up the entry by `entry_uid`; returns `not_a_tweet` if not found or wrong kind.
/// - Runs the scraper against the original `requested_locator`; if it fails (tweet deleted/private),
/// returns `scraper_failed` WITHOUT touching the existing data.
/// - On success: atomically deletes old entry_artifacts and re-registers all produced tweet JSONs
/// and their media blobs. Does not change archived_at, title, tags, or collections.
pub fn perform_rearchive(
archive_paths: &ArchivePaths,
entry_uid: &str,
config: &CaptureConfig,
) -> Result<RearchiveResult> {
let store_path = &archive_paths.store_path;
let mut conn = database::open_or_initialize(&archive_paths.archive_path)?;
// Look up the entry.
let entry = match database::get_entry_for_rearchive(&conn, entry_uid)? {
None => {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: "entry not found".to_string(),
});
}
Some(e) => e,
};
// Only tweet / tweet_thread entries can be re-archived this way.
if entry.entity_kind != "tweet" && entry.entity_kind != "tweet_thread" {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: format!("entry is '{}', not a tweet", entry.entity_kind),
});
}
// Parse source_metadata_json for tweet_id and requested_locator.
let meta: serde_json::Value = serde_json::from_str(&entry.source_metadata_json)
.unwrap_or(serde_json::Value::Object(Default::default()));
let requested_locator = meta["requested_locator"]
.as_str()
.unwrap_or("")
.to_string();
let tweet_id = meta["tweet_id"].as_str().unwrap_or("").to_string();
if requested_locator.is_empty() || tweet_id.is_empty() {
return Ok(RearchiveResult {
status: "not_a_tweet".to_string(),
message: "entry source_metadata_json missing tweet_id or requested_locator".to_string(),
});
}
let is_thread = entry.entity_kind == "tweet_thread";
let timestamp = format!(
"{}-{}",
chrono::Local::now().format("%Y-%m-%dT%H-%M-%S%.3f"),
uuid::Uuid::new_v4().simple(),
);
let tweet_cookies = resolve_cookies_for_url(&config.cookie_rules, "https://x.com/");
// Run the scraper (staged). If this fails, existing data is untouched.
let tweet_json_relpaths = match downloader::tweets::rearchive(
&requested_locator,
is_thread,
store_path,
&timestamp,
&tweet_cookies,
) {
Ok(relpaths) => relpaths,
Err(e) => {
return Ok(RearchiveResult {
status: "scraper_failed".to_string(),
message: format!("{e:#}"),
});
}
};
// Atomically swap artifact rows: delete old, insert new.
{
let tx = conn.transaction()?;
database::delete_entry_artifacts(&tx, entry.id)?;
register_tweet_artifacts(&tx, store_path, entry.id, &tweet_json_relpaths)?;
tx.commit()?;
}
database::refresh_entry_cached_bytes(&conn, entry.id)?;
eprintln!("info: rearchived entry {entry_uid}: {} tweet JSONs", tweet_json_relpaths.len());
Ok(RearchiveResult {
status: "completed".to_string(),
message: String::new(),
})
}
fn mime_for_font_ext(ext: &str) -> Option<String> {
match ext {
".woff2" => Some("font/woff2".to_string()),
@ -1988,6 +2109,7 @@ mod tests {
"tweet:123",
Source::Tweet,
"123",
&["raw_tweets/tweet-123.json".to_string()],
)
.unwrap();
database::finish_archive_run(&conn, run.id).unwrap();

37
crates/archivr-core/src/database.rs
View file

@ -2391,6 +2391,43 @@ fn humanize_slug(slug: &str) -> String {
.join(" ")
}
/// A minimal view of an archived entry needed for re-archiving.
pub struct EntryForRearchive {
pub id: i64,
pub entity_kind: String,
pub source_metadata_json: String,
}
/// Looks up an entry by its public UID for re-archive purposes.
pub fn get_entry_for_rearchive(
conn: &Connection,
entry_uid: &str,
) -> Result<Option<EntryForRearchive>> {
conn.query_row(
"SELECT id, entity_kind, source_metadata_json \
FROM archived_entries WHERE entry_uid = ?1",
[entry_uid],
|row| {
Ok(EntryForRearchive {
id: row.get(0)?,
entity_kind: row.get(1)?,
source_metadata_json: row.get(2)?,
})
},
)
.optional()
.map_err(anyhow::Error::from)
}
/// Deletes all `entry_artifacts` rows for the given entry.
/// Call within a transaction for atomicity with re-insertion.
pub fn delete_entry_artifacts(conn: &Connection, entry_id: i64) -> Result<usize> {
Ok(conn.execute(
"DELETE FROM entry_artifacts WHERE entry_id = ?1",
[entry_id],
)?)
}
#[cfg(test)]
mod tests {
use super::*;

199
crates/archivr-core/src/downloader/tweets.rs
View file

@ -1,7 +1,7 @@
use anyhow::{Context, Result, bail};
use regex::Regex;
use std::{
collections::{HashMap, HashSet},
collections::HashMap,
env,
ffi::OsString,
fs,
@ -59,63 +59,39 @@ fn build_scraper_args(
args
}
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
///
/// Invokes the Python scraper, then moves all produced media assets into the
/// content-addressed raw store and rewrites the JSON output to use the new
/// store-relative paths. Returns `true` if new content was archived, `false`
/// if the tweet was already present and `thread` is `false`.
///
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
pub fn archive(
path: &str,
/// Runs the scraper into a staging dir, rewrites assets into raw/, moves tweet JSONs
/// to `raw_tweets/`, and returns their store-relative relpaths (e.g. `"raw_tweets/tweet-123.json"`).
/// The staging dir starts empty so all files found there are exactly the touched set.
fn run_scraper_staged(
tweet_id: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<bool> {
) -> Result<Vec<String>> {
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
// Output directory for Tweet JSON files.
// staging_dir = store_path/temp/{timestamp}/tweet_stage/
// staging_tweets_dir = staging_dir/raw_tweets/ ← scraper writes JSONs here
// Scraper media-dir = staging_dir/media/
let staging_dir = store_path.join("temp").join(timestamp).join("tweet_stage");
let staging_tweets_dir = staging_dir.join("raw_tweets");
fs::create_dir_all(&staging_tweets_dir)?;
// Final destination for tweet JSONs.
let output_dir = store_path.join("raw_tweets");
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
fs::create_dir_all(&output_dir)?;
fs::create_dir_all(&temp_dir)?;
// Path to the root - the to-be-archived tweet's JSON file.
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
if !thread && root_json.exists() {
return Ok(false);
}
let before = tweet_json_files(&output_dir)?;
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
// Credentials: only use cookie rules as Twitter credentials when the resolved
// map contains both `ct0` AND `auth_token` — the two cookies the scraper
// requires. A global cookie rule for an unrelated site must not suppress the
// ARCHIVR_TWITTER_CREDENTIALS_FILE fallback.
// Fall back to ARCHIVR_TWITTER_CREDENTIALS_FILE otherwise.
// The temp file is written in the semicolon-delimited format the Python scraper
// expects (`ct0=val;auth_token=val`) and deleted unconditionally after the
// subprocess returns so secrets are never left on disk on failure.
let temp_creds_path: Option<PathBuf>;
let credentials_file: PathBuf;
let has_twitter_cookies =
cookies.contains_key("ct0") && cookies.contains_key("auth_token");
let has_twitter_cookies = cookies.contains_key("ct0") && cookies.contains_key("auth_token");
if has_twitter_cookies {
let cf = store_path.join("temp").join(timestamp).join("twitter-creds.txt");
// Semicolon-separated, no spaces — matches what the scraper's split(";") expects.
let creds_str = cookies
.iter()
.map(|(k, v)| format!("{k}={v}"))
@ -135,7 +111,7 @@ pub fn archive(
let mut f = std::fs::File::create(&cf)
.context("failed to write twitter credentials file")?;
f.write_all(creds_str.as_bytes())
.context("failed to write twitter credentials file")?;
.context("failed to write twitter credentials file")?
}
temp_creds_path = Some(cf.clone());
credentials_file = cf;
@ -156,28 +132,21 @@ pub fn archive(
}
let mut cmd = Command::new(&python);
cmd.current_dir(&temp_dir).arg(&scraper_path);
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
// Run the scraper from staging_dir so relative paths in JSON resolve correctly.
cmd.current_dir(&staging_dir).arg(&scraper_path);
for arg in build_scraper_args(tweet_id, thread, &staging_tweets_dir, &staging_dir, &credentials_file) {
cmd.arg(arg);
}
// Hold the Result so we can delete the credentials file before propagating
// any error — including spawn failures.
let spawn_result = cmd.output().with_context(|| {
format!(
"Failed to spawn tweet scraper at {}",
scraper_path.display()
)
format!("Failed to spawn tweet scraper at {}", scraper_path.display())
});
// Remove temp credentials file unconditionally — secrets must not persist.
if let Some(cf) = &temp_creds_path {
let _ = fs::remove_file(cf);
}
let output = spawn_result?;
if !output.status.success() {
let _ = fs::remove_dir_all(&staging_dir);
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
bail!(
@ -187,7 +156,9 @@ pub fn archive(
);
}
let root_json = staging_tweets_dir.join(format!("tweet-{tweet_id}.json"));
if !root_json.exists() {
let _ = fs::remove_dir_all(&staging_dir);
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
bail!(
@ -198,13 +169,82 @@ pub fn archive(
);
}
cleanup_summary(&output_dir)?;
let after = tweet_json_files(&output_dir)?;
let new_jsons = new_tweet_jsons(&before, &after);
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
// Remove the scraping_summary.json if the scraper left one.
cleanup_summary(&staging_tweets_dir)?;
Ok(true)
// Collect all tweet-*.json files from staging (this is the exact touched set).
let mut staged_jsons: Vec<PathBuf> = fs::read_dir(&staging_tweets_dir)?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with("tweet-") && n.ends_with(".json"))
})
.collect();
staged_jsons.sort();
// Rewrite asset paths in staged JSONs (moves media blobs from staging into raw/).
// staging_tweets_dir is base for avatar_local_path; staging_dir is base for local_path.
rewrite_tweet_outputs(&staged_jsons, &staging_tweets_dir, &staging_dir, store_path)?;
// Move each staged JSON to its final destination, collecting store-relative relpaths.
let mut relpaths = Vec::with_capacity(staged_jsons.len());
for staged_path in &staged_jsons {
let filename = staged_path.file_name().context("tweet JSON path has no filename")?;
let dest = output_dir.join(filename);
fs::rename(staged_path, &dest)
.or_else(|_| {
fs::copy(staged_path, &dest).map(|_| ())?;
fs::remove_file(staged_path)
})
.with_context(|| format!("failed to move staged tweet JSON to {}", dest.display()))?;
let rel = dest
.strip_prefix(store_path)
.context("dest tweet JSON is not under store_path")?;
relpaths.push(rel.to_string_lossy().replace('\\', "/"));
}
// Clean up the staging directory (media already moved to raw/ by rewrite_tweet_outputs).
let _ = fs::remove_dir_all(&staging_dir);
Ok(relpaths)
}
/// Archives a tweet (or full thread) identified by `path`.
///
/// Returns store-relative relpaths of all tweet JSON files registered for this capture.
/// For single tweets already on disk, returns the existing file path without re-downloading.
/// For new or thread captures, stages output in a temp dir then moves to `raw_tweets/`.
pub fn archive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<Vec<String>> {
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
let root_json = store_path.join("raw_tweets").join(format!("tweet-{tweet_id}.json"));
if !thread && root_json.exists() {
// Already present; skip re-download, return the existing file for artifact registration.
return Ok(vec![format!("raw_tweets/tweet-{tweet_id}.json")]);
}
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
}
/// Re-archives a tweet or thread, replacing files in `raw_tweets/` with fresh content.
/// Always runs the scraper; output is staged before replacing existing files.
/// Returns store-relative relpaths of all produced tweet JSON files.
/// If the scraper fails (tweet deleted/private), returns an error; existing files are untouched.
pub fn rearchive(
path: &str,
thread: bool,
store_path: &Path,
timestamp: &str,
cookies: &HashMap<String, String>,
) -> Result<Vec<String>> {
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
run_scraper_staged(&tweet_id, thread, store_path, timestamp, cookies)
}
/// Removes the `scraping_summary.json` file left by the scraper, if present.
@ -216,33 +256,6 @@ fn cleanup_summary(output_dir: &Path) -> Result<()> {
Ok(())
}
/// Returns the set of `tweet-*.json` files present in `output_dir`.
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
let mut files = HashSet::new();
for entry in fs::read_dir(output_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_file()
&& path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
{
files.insert(path);
}
}
Ok(files)
}
/// Returns the sorted list of JSON files present in `after` but not in `before`.
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
files.sort();
files
}
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
fn avatar_regex() -> &'static Regex {
@ -519,9 +532,9 @@ mod tests {
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
assert!(!archived);
assert_eq!(relpaths, vec!["raw_tweets/tweet-123.json"]);
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
let _ = fs::remove_dir_all(store_path);
@ -576,7 +589,7 @@ cat > "$output_dir/tweet-$tweet_id.json" <<EOF
{
"id": "$tweet_id",
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
"author": { "avatar_local_path": "$media_dir/avatars/author.jpg" }
}
EOF
"#,
@ -592,16 +605,16 @@ EOF
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
let archived = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
let relpaths = archive("tweet:123", false, &store_path, "ts", &HashMap::new()).unwrap();
let tweet_file = output_dir.join("tweet-123.json");
let contents = fs::read_to_string(&tweet_file).unwrap();
assert!(archived);
assert!(!relpaths.is_empty());
assert!(tweet_file.exists());
assert!(!output_dir.join("scraping_summary.json").exists());
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
assert!(contents.contains(r#""local_path": "raw/"#));
assert!(!store_path.join("temp").join("ts").exists());
assert!(!store_path.join("temp").join("ts").join("tweet_stage").exists());
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
remove_test_env("ARCHIVR_TWEET_SCRAPER");