mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat: server-side t.co resolver + frontend augmentation
Server (routes.rs): POST /api/util/resolve-tco — unauthenticated, accepts JSON array of https://t.co/<alphanumeric> URLs only (strict regex validation, no SSRF via input), capped at 50 per batch, 3 s timeout, redirect(Policy::none) so the server only ever touches t.co itself. HEAD first, GET fallback if HEAD returns no Location. Location sanitized to http/https only — javascript:/data:/etc. fall back to the original t.co. api.js: resolveTcoUrls(urls) — project-convention wrapper for the new endpoint. Returns {} on failure (callers degrade gracefully to bare t.co links). TweetPreview.jsx: After tweet data loads, per-tweet range-based coverage detection: builds covered [start,end) from existing entity fromIndex/toIndex or indices fallback, then finds regex matches whose span is NOT covered. Resolves unique uncovered t.co URLs via resolveTcoUrls(), synthesises one entity per occurrence (with exact fromIndex/toIndex so normalizeUrlAnn gets correct bounds even for duplicate t.co URLs in the same tweet). Augments entities.urls before setTweets() so all rendering paths see expanded URLs.
This commit is contained in:
parent
a06c541605
commit
1589eb68b5
9 changed files with 185 additions and 50 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -133,6 +133,7 @@ dependencies = [
|
|||
"parking_lot",
|
||||
"rand",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ serde_json.workspace = true
|
|||
rusqlite.workspace = true
|
||||
parking_lot.workspace = true
|
||||
regex.workspace = true
|
||||
reqwest.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
|
|
|||
|
|
@ -328,6 +328,7 @@ pub fn app_with_state(state: AppState) -> Router {
|
|||
"/api/archives/:archive_id/blob-cleanup",
|
||||
get(blob_cleanup_scan_handler).delete(blob_cleanup_delete_handler),
|
||||
)
|
||||
.route("/api/util/resolve-tco", post(resolve_tco_handler))
|
||||
.nest_service("/assets", ServeDir::new(static_dir.join("assets")))
|
||||
.fallback_service(ServeFile::new(static_dir.join("index.html")))
|
||||
.layer(axum::middleware::from_fn_with_state(state.clone(), setup_guard))
|
||||
|
|
@ -1908,6 +1909,74 @@ async fn delete_collection_handler(
|
|||
}
|
||||
}
|
||||
|
||||
// ── t.co resolver ──────────────────────────────────────────────────────────────
|
||||
// POST /api/util/resolve-tco
|
||||
// Body: JSON array of t.co short URLs (max 50).
|
||||
// Returns: JSON object mapping each input URL to its expanded destination.
|
||||
//
|
||||
// Security:
|
||||
// - Input restricted to https://t.co/<alphanumeric token> only (no SSRF via input).
|
||||
// - redirect(Policy::none()): makes ONE HEAD to t.co, reads Location header, never
|
||||
// fetches the expanded destination (no open-proxy).
|
||||
// - 3 s timeout, 1-hop max.
|
||||
// - No auth required (t.co is public; no data exposed).
|
||||
|
||||
static TCO_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
|
||||
regex::Regex::new(r"^https://t\.co/[A-Za-z0-9]+$").unwrap()
|
||||
});
|
||||
|
||||
async fn resolve_tco_handler(
|
||||
Json(urls): Json<Vec<String>>,
|
||||
) -> Result<Json<std::collections::HashMap<String, String>>, ApiError> {
|
||||
const MAX_BATCH: usize = 50;
|
||||
let urls: Vec<String> = urls
|
||||
.into_iter()
|
||||
.filter(|u| TCO_RE.is_match(u))
|
||||
.take(MAX_BATCH)
|
||||
.collect();
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()
|
||||
.map_err(|e| ApiError::internal(&format!("http client: {e}")))?;
|
||||
|
||||
let mut map = std::collections::HashMap::new();
|
||||
let futs: Vec<_> = urls.iter().map(|url| {
|
||||
let client = client.clone();
|
||||
let url = url.clone();
|
||||
tokio::spawn(async move {
|
||||
// Try HEAD first; fall back to GET if HEAD returns no Location.
|
||||
// Neither follows redirects (Policy::none), so the server only
|
||||
// ever connects to t.co itself — never to the destination.
|
||||
// Only accept http/https destinations — never javascript:, data:, etc.
|
||||
let safe_location = |resp: reqwest::Response| {
|
||||
resp.headers()
|
||||
.get(reqwest::header::LOCATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string())
|
||||
.filter(|s| s.starts_with("http://") || s.starts_with("https://"))
|
||||
};
|
||||
let expanded = match client.head(&url).send().await.ok().and_then(safe_location) {
|
||||
Some(loc) => loc,
|
||||
None => match client.get(&url).send().await.ok().and_then(safe_location) {
|
||||
Some(loc) => loc,
|
||||
None => url.clone(),
|
||||
},
|
||||
};
|
||||
(url, expanded)
|
||||
})
|
||||
}).collect();
|
||||
|
||||
for fut in futs {
|
||||
if let Ok((k, v)) = fut.await {
|
||||
map.insert(k, v);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Json(map))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
47
crates/archivr-server/static/assets/index-CBWT1bsB.js
Normal file
47
crates/archivr-server/static/assets/index-CBWT1bsB.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4,7 +4,7 @@
|
|||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Archivr</title>
|
||||
<script type="module" crossorigin src="/assets/index-WdvYwWGH.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-CBWT1bsB.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-C89vW3ol.css">
|
||||
</head>
|
||||
<body>
|
||||
|
|
|
|||
|
|
@ -35,6 +35,20 @@ export function fetchEntryArtifacts(archiveId, entryUid, indices) {
|
|||
);
|
||||
}
|
||||
|
||||
// Resolve t.co short URLs server-side (HEAD→GET, no-follow).
|
||||
// Returns a map { [tcoUrl]: expandedUrl }.
|
||||
// Silently returns {} on failure — callers fall back to plain t.co links.
|
||||
export async function resolveTcoUrls(urls) {
|
||||
if (!urls || urls.length === 0) return {};
|
||||
const res = await fetch('/api/util/resolve-tco', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(urls),
|
||||
});
|
||||
if (!res.ok) return {};
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function updateEntryTitle(archiveId, entryUid, title) {
|
||||
const res = await fetch(`/api/archives/${archiveId}/entries/${entryUid}`, {
|
||||
method: 'PATCH',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useState, useEffect } from 'react';
|
||||
import { fetchEntryArtifacts } from '../api';
|
||||
import { fetchEntryArtifacts, resolveTcoUrls } from '../api';
|
||||
|
||||
// ── Date helper ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -1148,7 +1148,57 @@ export default function TweetPreview({ archiveId, entryUid, artifacts, entityKin
|
|||
let cancelled = false;
|
||||
|
||||
fetchEntryArtifacts(archiveId, entryUid, tweetArtifacts.map(a => a.index))
|
||||
.then(data => { if (!cancelled) setTweets(data); })
|
||||
.then(async data => {
|
||||
if (cancelled) return;
|
||||
|
||||
const tcoRe = /https?:\/\/t\.co\/[A-Za-z0-9]+/g;
|
||||
|
||||
// For each tweet, build the set of [start,end) spans already covered by
|
||||
// existing URL entities. A regex match is only "entity-less" if its span
|
||||
// is not covered — avoids skipping real duplicate occurrences.
|
||||
const coveredRanges = tweet => {
|
||||
const ft = tweet.full_text || '';
|
||||
return (tweet.entities?.urls || []).map(u => {
|
||||
if (u.fromIndex != null && u.toIndex != null) return [u.fromIndex, u.toIndex];
|
||||
if (u.indices?.length === 2) return [u.indices[0], u.indices[1]];
|
||||
if (u.url) { const i = ft.indexOf(u.url); if (i !== -1) return [i, i + u.url.length]; }
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
};
|
||||
const isCovered = (ranges, s, e) => ranges.some(([cs, ce]) => cs <= s && ce >= e);
|
||||
|
||||
const uniqueTco = new Set();
|
||||
for (const t of data) {
|
||||
const ft = t.full_text || '';
|
||||
const ranges = coveredRanges(t);
|
||||
let m; tcoRe.lastIndex = 0;
|
||||
while ((m = tcoRe.exec(ft)) !== null) {
|
||||
if (!isCovered(ranges, m.index, m.index + m[0].length)) uniqueTco.add(m[0]);
|
||||
}
|
||||
}
|
||||
|
||||
const resolved = uniqueTco.size > 0 ? await resolveTcoUrls([...uniqueTco]).catch(() => ({})) : {};
|
||||
|
||||
const augmented = data.map(t => {
|
||||
const ft = t.full_text || '';
|
||||
const ranges = coveredRanges(t);
|
||||
const synth = [];
|
||||
let m; tcoRe.lastIndex = 0;
|
||||
while ((m = tcoRe.exec(ft)) !== null) {
|
||||
const tco = m[0]; const exp = resolved[tco];
|
||||
if (exp && exp !== tco && !isCovered(ranges, m.index, m.index + tco.length)) {
|
||||
synth.push({
|
||||
url: tco, expanded_url: exp,
|
||||
display_url: (() => { try { const u = new URL(exp); return u.hostname + (u.pathname.length > 1 ? '/…' : ''); } catch (_) { return exp; } })(),
|
||||
fromIndex: m.index, toIndex: m.index + tco.length,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (synth.length === 0) return t;
|
||||
return { ...t, entities: { ...(t.entities || {}), urls: [...(t.entities?.urls || []), ...synth] } };
|
||||
});
|
||||
if (!cancelled) setTweets(augmented);
|
||||
})
|
||||
.catch(e => { if (!cancelled) setError(e.message || 'Failed to load tweet.'); })
|
||||
.finally(() => { if (!cancelled) setLoading(false); });
|
||||
|
||||
|
|
|
|||
BIN
vendor/twitter/__pycache__/scrape_user_tweet_contents.cpython-313.pyc
vendored
Normal file
BIN
vendor/twitter/__pycache__/scrape_user_tweet_contents.cpython-313.pyc
vendored
Normal file
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue