crawler: add crawler

hosts/thegeneralist-central/archive/archivebox.nix

@@ -0,0 +1,34 @@
+{ pkgs, ... }:
+{
+  # virtualisation.docker.enable = true;
+  virtualisation.podman = {
+    enable = true;
+    dockerCompat = true;
+  };
+  virtualisation.oci-containers.containers = {
+    archivebox = {
+      image = "ghcr.io/archivebox/archivebox:main";
+      ports = [ "127.0.0.1:8000:8000" ];
+      volumes = [
+        "/mnt/usb/services/archivebox/data:/data"
+      ];
+      environment = {
+        ALLOWLIST_HOSTS = "localhost";
+        CSRF_TRUSTED_ORIGINS = "https://archive.thegeneralist01.com,127.0.0.1:8000";
+        REVERSE_PROXY_USER_HEADER = "X-Remote-User";
+        REVERSE_PROXY_WHITELIST = "127.0.0.1/32,100.86.129.23/32";
+      };
+    };
+
+    pywb = {
+      image = "docker.io/webrecorder/pywb";
+      ports = [ "127.0.0.1:8001:8001" ];
+      volumes = [
+        "/mnt/usb/services/browsertrix/webrecorder/:/"
+        "/mnt/usb/services/browsertrix/webrecorder/webarchive:/webarchive"
+      ];
+    };
+  };
+
+  environment.systemPackages = [ pkgs.docker ];
+}

hosts/thegeneralist-central/archive/crawler-site.nix

@@ -0,0 +1,41 @@
+let
+  acmeDomain = "thegeneralist01.com";
+  domain = "crawler.${acmeDomain}";
+
+  ssl = {
+    forceSSL = true;
+    quic = true;
+    useACMEHost = domain;
+  };
+in
+{
+  services.nginx.virtualHosts.${domain} = ssl // {
+    listen = [
+      {
+        addr = "100.86.129.23";
+        port = 443;
+        ssl = true;
+      }
+      {
+        addr = "100.86.129.23";
+        port = 80;
+      }
+    ];
+
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:8001";
+      recommendedProxySettings = true;
+      extraConfig = ''
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+
+        # tell nginx not to buffer the response. send it as it comes.
+        proxy_buffering off;
+
+        # give jellyfin plenty of time to transcode
+        proxy_read_timeout 3600s;
+        proxy_send_timeout 3600s;
+      '';
+    };
+  };
+}

hosts/thegeneralist-central/archive/default.nix

@@ -9,6 +9,8 @@ let
   };
 in
 {
+  imports = [ ./archivebox.nix ./crawler-site.nix ];
+
   services.nginx.virtualHosts.${domain} = ssl // {
     listen = [
       {