thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-05-30 08:36:47 +02:00
Code Issues Activity

Add Twitter tweet and thread archiving support

Browse source
This commit is contained in:
TheGeneralist 2026-03-31 21:25:24 +02:00
parent 9441a9d9fb
commit 81c373ca8f
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
7 changed files with 1738 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

74
flake.nix
View file

@ -29,6 +29,37 @@
system:
let
pkgs = import nixpkgs { inherit system; };
pyPkgs = pkgs.python312Packages;
twitterApiClient = pyPkgs.buildPythonPackage rec {
pname = "twitter-api-client";
version = "0.10.22";
format = "setuptools";
src = pkgs.fetchPypi {
pname = "twitter_api_client";
inherit version;
hash = "sha256-S5KzQRDIQroc2bJsPLaKR9xocHKniqd9Z055CsC5rbQ=";
};
nativeBuildInputs = [ pyPkgs.setuptools pyPkgs.wheel ];
propagatedBuildInputs = [
pyPkgs.aiofiles
pyPkgs."nest-asyncio"
pyPkgs.httpx
pyPkgs.tqdm
pyPkgs.orjson
pyPkgs.m3u8
pyPkgs.websockets
pyPkgs.uvloop
];
pythonImportsCheck = [ "twitter" ];
doCheck = false;
};
tweetPython = pkgs.python312.withPackages (
ps: [
ps.tomlkit
ps."tomli-w"
twitterApiClient
]
);
archivr_unwrapped = pkgs.rustPlatform.buildRustPackage {
pname = "archivr";
version = "0.1.0";
@ -42,18 +73,24 @@
nativeBuildInputs = [ pkgs.makeWrapper ];
buildInputs = [
pkgs.yt-dlp
tweetPython
];
phases = [ "installPhase" ];
installPhase = ''
mkdir -p $out/bin
mkdir -p $out/bin $out/libexec/archivr
cp -r ${archivr_unwrapped}/bin/* $out/bin/
cp ${./vendor/twitter/scrape_user_tweet_contents.py} $out/libexec/archivr/scrape_user_tweet_contents.py
chmod +x $out/libexec/archivr/scrape_user_tweet_contents.py
for f in $out/bin/*; do
mv "$f" "$f.orig"
makeWrapper "$f.orig" "$f" \
--set ARCHIVR_YT_DLP ${pkgs.yt-dlp}/bin/yt-dlp \
--set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \
--set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr/scrape_user_tweet_contents.py \
--prefix PATH : ${
lib.makeBinPath [
pkgs.yt-dlp
tweetPython
]
}
done
@ -71,16 +108,49 @@
system:
let
pkgs = import nixpkgs { inherit system; };
pyPkgs = pkgs.python312Packages;
twitterApiClient = pyPkgs.buildPythonPackage rec {
pname = "twitter-api-client";
version = "0.10.22";
format = "setuptools";
src = pkgs.fetchPypi {
pname = "twitter_api_client";
inherit version;
hash = "sha256-S5KzQRDIQroc2bJsPLaKR9xocHKniqd9Z055CsC5rbQ=";
};
nativeBuildInputs = [ pyPkgs.setuptools pyPkgs.wheel ];
propagatedBuildInputs = [
pyPkgs.aiofiles
pyPkgs."nest-asyncio"
pyPkgs.httpx
pyPkgs.tqdm
pyPkgs.orjson
pyPkgs.m3u8
pyPkgs.websockets
pyPkgs.uvloop
];
pythonImportsCheck = [ "twitter" ];
doCheck = false;
};
tweetPython = pkgs.python312.withPackages (
ps: [
ps.tomlkit
ps."tomli-w"
twitterApiClient
]
);
in
{
default = pkgs.mkShell {
buildInputs = [
pkgs.yt-dlp
pkgs.nushell
pkgs.uv
tweetPython
];
shellHook = ''
export SHELL=${pkgs.nushell}/bin/nu
echo "nushell dev shell active – yt-dlp on PATH"
echo "nushell dev shell active – yt-dlp, uv, and tweet scraper Python on PATH"
nu
'';
};