1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00

feat: server-side t.co resolver + frontend augmentation

Server (routes.rs):
  POST /api/util/resolve-tco — unauthenticated, accepts JSON array of
  https://t.co/<alphanumeric> URLs only (strict regex validation, no SSRF
  via input), capped at 50 per batch, 3 s timeout, redirect(Policy::none)
  so the server only ever touches t.co itself. HEAD first, GET fallback if
  HEAD returns no Location. Location sanitized to http/https only —
  javascript:/data:/etc. fall back to the original t.co.

api.js:
  resolveTcoUrls(urls) — project-convention wrapper for the new endpoint.
  Returns {} on failure (callers degrade gracefully to bare t.co links).

TweetPreview.jsx:
  After tweet data loads, per-tweet range-based coverage detection:
  builds covered [start,end) from existing entity fromIndex/toIndex or
  indices fallback, then finds regex matches whose span is NOT covered.
  Resolves unique uncovered t.co URLs via resolveTcoUrls(), synthesises
  one entity per occurrence (with exact fromIndex/toIndex so normalizeUrlAnn
  gets correct bounds even for duplicate t.co URLs in the same tweet).
  Augments entities.urls before setTweets() so all rendering paths
  see expanded URLs.
This commit is contained in:
TheGeneralist 2026-07-12 15:45:02 +02:00
parent a06c541605
commit 1589eb68b5
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
9 changed files with 185 additions and 50 deletions

1
Cargo.lock generated
View file

@ -133,6 +133,7 @@ dependencies = [
"parking_lot",
"rand",
"regex",
"reqwest",
"rusqlite",
"serde",
"serde_json",

View file

@ -21,6 +21,7 @@ serde_json.workspace = true
rusqlite.workspace = true
parking_lot.workspace = true
regex.workspace = true
reqwest.workspace = true
[dev-dependencies]
tempfile.workspace = true

View file

@ -328,6 +328,7 @@ pub fn app_with_state(state: AppState) -> Router {
"/api/archives/:archive_id/blob-cleanup",
get(blob_cleanup_scan_handler).delete(blob_cleanup_delete_handler),
)
.route("/api/util/resolve-tco", post(resolve_tco_handler))
.nest_service("/assets", ServeDir::new(static_dir.join("assets")))
.fallback_service(ServeFile::new(static_dir.join("index.html")))
.layer(axum::middleware::from_fn_with_state(state.clone(), setup_guard))
@ -1908,6 +1909,74 @@ async fn delete_collection_handler(
}
}
// ── t.co resolver ──────────────────────────────────────────────────────────────
// POST /api/util/resolve-tco
// Body: JSON array of t.co short URLs (max 50).
// Returns: JSON object mapping each input URL to its expanded destination.
//
// Security:
// - Input restricted to https://t.co/<alphanumeric token> only (no SSRF via input).
// - redirect(Policy::none()): makes ONE HEAD to t.co, reads Location header, never
// fetches the expanded destination (no open-proxy).
// - 3 s timeout, 1-hop max.
// - No auth required (t.co is public; no data exposed).
static TCO_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
regex::Regex::new(r"^https://t\.co/[A-Za-z0-9]+$").unwrap()
});
async fn resolve_tco_handler(
Json(urls): Json<Vec<String>>,
) -> Result<Json<std::collections::HashMap<String, String>>, ApiError> {
const MAX_BATCH: usize = 50;
let urls: Vec<String> = urls
.into_iter()
.filter(|u| TCO_RE.is_match(u))
.take(MAX_BATCH)
.collect();
let client = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.timeout(Duration::from_secs(3))
.build()
.map_err(|e| ApiError::internal(&format!("http client: {e}")))?;
let mut map = std::collections::HashMap::new();
let futs: Vec<_> = urls.iter().map(|url| {
let client = client.clone();
let url = url.clone();
tokio::spawn(async move {
// Try HEAD first; fall back to GET if HEAD returns no Location.
// Neither follows redirects (Policy::none), so the server only
// ever connects to t.co itself — never to the destination.
// Only accept http/https destinations — never javascript:, data:, etc.
let safe_location = |resp: reqwest::Response| {
resp.headers()
.get(reqwest::header::LOCATION)
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.filter(|s| s.starts_with("http://") || s.starts_with("https://"))
};
let expanded = match client.head(&url).send().await.ok().and_then(safe_location) {
Some(loc) => loc,
None => match client.get(&url).send().await.ok().and_then(safe_location) {
Some(loc) => loc,
None => url.clone(),
},
};
(url, expanded)
})
}).collect();
for fut in futs {
if let Ok((k, v)) = fut.await {
map.insert(k, v);
}
}
Ok(Json(map))
}
#[cfg(test)]
mod tests {
use super::*;

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,7 +4,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title>
<script type="module" crossorigin src="/assets/index-WdvYwWGH.js"></script>
<script type="module" crossorigin src="/assets/index-CBWT1bsB.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-C89vW3ol.css">
</head>
<body>

View file

@ -35,6 +35,20 @@ export function fetchEntryArtifacts(archiveId, entryUid, indices) {
);
}
// Resolve t.co short URLs server-side (HEAD→GET, no-follow).
// Returns a map { [tcoUrl]: expandedUrl }.
// Silently returns {} on failure — callers fall back to plain t.co links.
export async function resolveTcoUrls(urls) {
if (!urls || urls.length === 0) return {};
const res = await fetch('/api/util/resolve-tco', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(urls),
});
if (!res.ok) return {};
return res.json();
}
export async function updateEntryTitle(archiveId, entryUid, title) {
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}`, {
method: 'PATCH',

View file

@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import { fetchEntryArtifacts } from '../api';
import { fetchEntryArtifacts, resolveTcoUrls } from '../api';
// Date helper
@ -1148,7 +1148,57 @@ export default function TweetPreview({ archiveId, entryUid, artifacts, entityKin
let cancelled = false;
fetchEntryArtifacts(archiveId, entryUid, tweetArtifacts.map(a => a.index))
.then(data => { if (!cancelled) setTweets(data); })
.then(async data => {
if (cancelled) return;
const tcoRe = /https?:\/\/t\.co\/[A-Za-z0-9]+/g;
// For each tweet, build the set of [start,end) spans already covered by
// existing URL entities. A regex match is only "entity-less" if its span
// is not covered avoids skipping real duplicate occurrences.
const coveredRanges = tweet => {
const ft = tweet.full_text || '';
return (tweet.entities?.urls || []).map(u => {
if (u.fromIndex != null && u.toIndex != null) return [u.fromIndex, u.toIndex];
if (u.indices?.length === 2) return [u.indices[0], u.indices[1]];
if (u.url) { const i = ft.indexOf(u.url); if (i !== -1) return [i, i + u.url.length]; }
return null;
}).filter(Boolean);
};
const isCovered = (ranges, s, e) => ranges.some(([cs, ce]) => cs <= s && ce >= e);
const uniqueTco = new Set();
for (const t of data) {
const ft = t.full_text || '';
const ranges = coveredRanges(t);
let m; tcoRe.lastIndex = 0;
while ((m = tcoRe.exec(ft)) !== null) {
if (!isCovered(ranges, m.index, m.index + m[0].length)) uniqueTco.add(m[0]);
}
}
const resolved = uniqueTco.size > 0 ? await resolveTcoUrls([...uniqueTco]).catch(() => ({})) : {};
const augmented = data.map(t => {
const ft = t.full_text || '';
const ranges = coveredRanges(t);
const synth = [];
let m; tcoRe.lastIndex = 0;
while ((m = tcoRe.exec(ft)) !== null) {
const tco = m[0]; const exp = resolved[tco];
if (exp && exp !== tco && !isCovered(ranges, m.index, m.index + tco.length)) {
synth.push({
url: tco, expanded_url: exp,
display_url: (() => { try { const u = new URL(exp); return u.hostname + (u.pathname.length > 1 ? '/…' : ''); } catch (_) { return exp; } })(),
fromIndex: m.index, toIndex: m.index + tco.length,
});
}
}
if (synth.length === 0) return t;
return { ...t, entities: { ...(t.entities || {}), urls: [...(t.entities?.urls || []), ...synth] } };
});
if (!cancelled) setTweets(augmented);
})
.catch(e => { if (!cancelled) setError(e.message || 'Failed to load tweet.'); })
.finally(() => { if (!cancelled) setLoading(false); });