1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 03:05:32 +02:00

feat: server-side t.co resolver + frontend augmentation

Server (routes.rs):
  POST /api/util/resolve-tco — unauthenticated, accepts JSON array of
  https://t.co/<alphanumeric> URLs only (strict regex validation, no SSRF
  via input), capped at 50 per batch, 3 s timeout, redirect(Policy::none)
  so the server only ever touches t.co itself. HEAD first, GET fallback if
  HEAD returns no Location. Location sanitized to http/https only —
  javascript:/data:/etc. fall back to the original t.co.

api.js:
  resolveTcoUrls(urls) — project-convention wrapper for the new endpoint.
  Returns {} on failure (callers degrade gracefully to bare t.co links).

TweetPreview.jsx:
  After tweet data loads, per-tweet range-based coverage detection:
  builds covered [start,end) from existing entity fromIndex/toIndex or
  indices fallback, then finds regex matches whose span is NOT covered.
  Resolves unique uncovered t.co URLs via resolveTcoUrls(), synthesises
  one entity per occurrence (with exact fromIndex/toIndex so normalizeUrlAnn
  gets correct bounds even for duplicate t.co URLs in the same tweet).
  Augments entities.urls before setTweets() so all rendering paths
  see expanded URLs.
This commit is contained in:
TheGeneralist 2026-07-12 15:45:02 +02:00
parent a06c541605
commit 1589eb68b5
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
9 changed files with 185 additions and 50 deletions

View file

@ -21,6 +21,7 @@ serde_json.workspace = true
rusqlite.workspace = true
parking_lot.workspace = true
regex.workspace = true
reqwest.workspace = true
[dev-dependencies]
tempfile.workspace = true

View file

@ -328,6 +328,7 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/archives/:archive_id/blob-cleanup",
get(blob_cleanup_scan_handler).delete(blob_cleanup_delete_handler),
)
.route("/api/util/resolve-tco", post(resolve_tco_handler))
.nest_service("/assets", ServeDir::new(static_dir.join("assets")))
.fallback_service(ServeFile::new(static_dir.join("index.html")))
.layer(axum::middleware::from_fn_with_state(state.clone(), setup_guard))
@ -1908,6 +1909,74 @@ async fn delete_collection_handler(
}
}
// ── t.co resolver ──────────────────────────────────────────────────────────────
// POST /api/util/resolve-tco
// Body: JSON array of t.co short URLs (max 50).
// Returns: JSON object mapping each input URL to its expanded destination.
//
// Security:
// - Input restricted to https://t.co/<alphanumeric token> only (no SSRF via input).
// - redirect(Policy::none()): makes ONE HEAD to t.co, reads Location header, never
// fetches the expanded destination (no open-proxy).
// - 3 s timeout, 1-hop max.
// - No auth required (t.co is public; no data exposed).
static TCO_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
regex::Regex::new(r"^https://t\.co/[A-Za-z0-9]+$").unwrap()
});
async fn resolve_tco_handler(
Json(urls): Json<Vec<String>>,
) -> Result<Json<std::collections::HashMap<String, String>>, ApiError> {
const MAX_BATCH: usize = 50;
let urls: Vec<String> = urls
.into_iter()
.filter(|u| TCO_RE.is_match(u))
.take(MAX_BATCH)
.collect();
let client = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.timeout(Duration::from_secs(3))
.build()
.map_err(|e| ApiError::internal(&format!("http client: {e}")))?;
let mut map = std::collections::HashMap::new();
let futs: Vec<_> = urls.iter().map(|url| {
let client = client.clone();
let url = url.clone();
tokio::spawn(async move {
// Try HEAD first; fall back to GET if HEAD returns no Location.
// Neither follows redirects (Policy::none), so the server only
// ever connects to t.co itself — never to the destination.
// Only accept http/https destinations — never javascript:, data:, etc.
let safe_location = |resp: reqwest::Response| {
resp.headers()
.get(reqwest::header::LOCATION)
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.filter(|s| s.starts_with("http://") || s.starts_with("https://"))
};
let expanded = match client.head(&url).send().await.ok().and_then(safe_location) {
Some(loc) => loc,
None => match client.get(&url).send().await.ok().and_then(safe_location) {
Some(loc) => loc,
None => url.clone(),
},
};
(url, expanded)
})
}).collect();
for fut in futs {
if let Ok((k, v)) = fut.await {
map.insert(k, v);
}
}
Ok(Json(map))
}
#[cfg(test)]
mod tests {
use super::*;

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,7 +4,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title>
<script type="module" crossorigin src="/assets/index-WdvYwWGH.js"></script>
<script type="module" crossorigin src="/assets/index-CBWT1bsB.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-C89vW3ol.css">
</head>
<body>