thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-05-30 08:36:47 +02:00
Code Issues Activity

feat: add Twitter tweet/thread archiving and platform shorthand support (#5)

Browse source 
* Add Twitter tweet and thread archiving support

* Fix tweet scraper path resolution and error reporting

* Flatten tweet archives and rearchive tweet assets

* refactor: simplify archive source parsing

* Refactor tweet archive source handling

* Clean up some clanker-written code

Signed-off-by: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com>

* Rename resolve_from_cwd to absolutize_path

Update call sites and tests to use the new API. Adjust tweet scraper
path/credentials handling and make small tweaks to local path hashing
and
raw store helpers.

Signed-off-by: TheGeneralist
<180094941+thegeneralist01@users.noreply.github.com>
Signed-off-by: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com>

* Add docs for supported platforms, shorthands, and env vars

* Minor clean up

* Implement social shorthand URL expansion and tweet alias parsing

* Extract store and Twitter helpers into shared modules

---------

Signed-off-by: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com>
This commit is contained in:
TheGeneralist 2026-04-03 15:34:26 +02:00 • committed by GitHub
parent 9441a9d9fb
commit 6bee5adfdd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 2377 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

74
flake.nix
View file

@ -29,6 +29,37 @@
system:
let
pkgs = import nixpkgs { inherit system; };
pyPkgs = pkgs.python312Packages;
twitterApiClient = pyPkgs.buildPythonPackage rec {
pname = "twitter-api-client";
version = "0.10.22";
format = "setuptools";
src = pkgs.fetchPypi {
pname = "twitter_api_client";
inherit version;
hash = "sha256-S5KzQRDIQroc2bJsPLaKR9xocHKniqd9Z055CsC5rbQ=";
};
nativeBuildInputs = [ pyPkgs.setuptools pyPkgs.wheel ];
propagatedBuildInputs = [
pyPkgs.aiofiles
pyPkgs."nest-asyncio"
pyPkgs.httpx
pyPkgs.tqdm
pyPkgs.orjson
pyPkgs.m3u8
pyPkgs.websockets
pyPkgs.uvloop
];
pythonImportsCheck = [ "twitter" ];
doCheck = false;
};
tweetPython = pkgs.python312.withPackages (
ps: [
ps.tomlkit
ps."tomli-w"
twitterApiClient
]
);
archivr_unwrapped = pkgs.rustPlatform.buildRustPackage {
pname = "archivr";
version = "0.1.0";
@ -42,18 +73,24 @@
nativeBuildInputs = [ pkgs.makeWrapper ];
buildInputs = [
pkgs.yt-dlp
tweetPython
];
phases = [ "installPhase" ];
installPhase = ''
mkdir -p $out/bin
mkdir -p $out/bin $out/libexec/archivr
cp -r ${archivr_unwrapped}/bin/* $out/bin/
cp ${./vendor/twitter/scrape_user_tweet_contents.py} $out/libexec/archivr/scrape_user_tweet_contents.py
chmod +x $out/libexec/archivr/scrape_user_tweet_contents.py
for f in $out/bin/*; do
mv "$f" "$f.orig"
makeWrapper "$f.orig" "$f" \
--set ARCHIVR_YT_DLP ${pkgs.yt-dlp}/bin/yt-dlp \
--set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \
--set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr/scrape_user_tweet_contents.py \
--prefix PATH : ${
lib.makeBinPath [
pkgs.yt-dlp
tweetPython
]
}
done
@ -71,16 +108,49 @@
system:
let
pkgs = import nixpkgs { inherit system; };
pyPkgs = pkgs.python312Packages;
twitterApiClient = pyPkgs.buildPythonPackage rec {
pname = "twitter-api-client";
version = "0.10.22";
format = "setuptools";
src = pkgs.fetchPypi {
pname = "twitter_api_client";
inherit version;
hash = "sha256-S5KzQRDIQroc2bJsPLaKR9xocHKniqd9Z055CsC5rbQ=";
};
nativeBuildInputs = [ pyPkgs.setuptools pyPkgs.wheel ];
propagatedBuildInputs = [
pyPkgs.aiofiles
pyPkgs."nest-asyncio"
pyPkgs.httpx
pyPkgs.tqdm
pyPkgs.orjson
pyPkgs.m3u8
pyPkgs.websockets
pyPkgs.uvloop
];
pythonImportsCheck = [ "twitter" ];
doCheck = false;
};
tweetPython = pkgs.python312.withPackages (
ps: [
ps.tomlkit
ps."tomli-w"
twitterApiClient
]
);
in
{
default = pkgs.mkShell {
buildInputs = [
pkgs.yt-dlp
pkgs.nushell
pkgs.uv
tweetPython
];
shellHook = ''
export SHELL=${pkgs.nushell}/bin/nu
echo "nushell dev shell active – yt-dlp on PATH"
echo "nushell dev shell active – yt-dlp, uv, and tweet scraper Python on PATH"
nu
'';
};