mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
feat: expand t.co links; linkify bare URLs in tweet and article text
Frontend: - resolveEntityBounds: try multiple candidate strings in order (u.url first, since that's the t.co short URL that appears in full_text) - normalizeUrlAnn: multi-candidate search; href = expanded > url, display = display_url > expanded > url - linkifyText(): regex linkifier for entity-less bare URLs; trims trailing punctuation [.,;:!?)] before linking; used in both renderTweetTextJSX and renderInlineJSX including their early-return paths (anns.length === 0) that previously bypassed linkification - renderInlineJSX: fix mention href mention.name → screen_name; replace t.co segment text with url.display when entity covers it Scraper (vendor/twitter/scrape_user_tweet_contents.py): - extract_tweet_data: when note_tweet text is used, pull urls/mentions/ hashtags/symbols from note_result.entity_set (correct indices for the note text); keep media from legacy.entities (no note media downloads)
This commit is contained in:
parent
900e33fa60
commit
a06c541605
4 changed files with 113 additions and 50 deletions
23
vendor/twitter/scrape_user_tweet_contents.py
vendored
23
vendor/twitter/scrape_user_tweet_contents.py
vendored
|
|
@ -493,14 +493,23 @@ def extract_tweet_data(
|
|||
# Extract is_quote_status (bare)
|
||||
tweet_data["is_quote_status"] = legacy.get("is_quote_status", False)
|
||||
|
||||
# Extract entities (always included)
|
||||
entities = legacy.get("entities", {})
|
||||
# Extract entities - when note_tweet text is used, its entity_set has the
|
||||
# correct indices for that text; legacy.entities indices match legacy.full_text
|
||||
# (truncated) and will be wrong/missing for long tweets.
|
||||
note_result = (
|
||||
tweet_result.get("note_tweet", {})
|
||||
.get("note_tweet_results", {})
|
||||
.get("result", {})
|
||||
)
|
||||
note_ents = note_result.get("entity_set", {}) if note_tweet_text else {}
|
||||
legacy_ents = legacy.get("entities", {})
|
||||
tweet_data["entities"] = {
|
||||
"hashtags": entities.get("hashtags", []),
|
||||
"urls": entities.get("urls", []),
|
||||
"user_mentions": entities.get("user_mentions", []),
|
||||
"symbols": entities.get("symbols", []),
|
||||
"media": entities.get("media", []) if not bare_scrape else [],
|
||||
"hashtags": note_ents.get("hashtags", legacy_ents.get("hashtags", [])),
|
||||
"urls": note_ents.get("urls", legacy_ents.get("urls", [])),
|
||||
"user_mentions": note_ents.get("user_mentions", legacy_ents.get("user_mentions", [])),
|
||||
"symbols": note_ents.get("symbols", legacy_ents.get("symbols", [])),
|
||||
# media always from legacy (note_tweet has no media downloads)
|
||||
"media": legacy_ents.get("media", []) if not bare_scrape else [],
|
||||
}
|
||||
|
||||
# Extract optional fields if not bare scrape
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue