From 38d3066ee42abc2016121b70105151a65ca82163 Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:20:55 +0200 Subject: [PATCH 1/3] fix: address code review issues with Docker setup - .gitignore: whitelist Dockerfile, docker-compose.yml, docker/ so they are actually tracked (the * catch-all was silently dropping them) - Dockerfile: build and ship the archivr CLI alongside archivr-server so users can run `archivr init` inside the container on first setup - docker/config.example.toml: fix archive_path to point at the .archivr subdirectory that archivr init creates (not the parent directory), which is what read_archive_paths expects - docs/README.md: replace the bare mkdir quickstart step with `archivr init`, explain why mkdir is insufficient; add a callout that auth_db_path must be set explicitly to a writable path when the config mount is read-only --- .gitignore | 5 ++ Dockerfile | 97 ++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 23 +++++++++ docker/config.example.toml | 32 +++++++++++++ docs/README.md | 16 +++++-- 5 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker/config.example.toml diff --git a/.gitignore b/.gitignore index 754d1d4..7dbca8c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,11 @@ !ARCHIVR-MENTAL-MODEL.md !NEXT.md +!Dockerfile +!docker-compose.yml +!docker/ +!docker/** + !modules/ !modules/** diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6a451d2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,97 @@ +# syntax=docker/dockerfile:1 + +############################################################################### +# Stage 1 – Build the Rust server and CLI binaries +############################################################################### +FROM rust:1.87-slim-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Layer the dependency build separately for better cache reuse. +# Stub out every crate so Cargo can resolve and compile all dependencies +# before we copy the real source. +COPY Cargo.toml Cargo.lock ./ +COPY crates/archivr-core/Cargo.toml crates/archivr-core/Cargo.toml +COPY crates/archivr-server/Cargo.toml crates/archivr-server/Cargo.toml +COPY crates/archivr-cli/Cargo.toml crates/archivr-cli/Cargo.toml + +RUN mkdir -p \ + crates/archivr-core/src \ + crates/archivr-server/src \ + crates/archivr-cli/src \ + && touch crates/archivr-core/src/lib.rs \ + && echo 'fn main() {}' > crates/archivr-server/src/main.rs \ + && echo 'fn main() {}' > crates/archivr-cli/src/main.rs \ + && cargo build --release -p archivr-server -p archivr-cli || true + +# Build the real binaries; touch source files to force Cargo to relink. +COPY crates/ crates/ +RUN touch \ + crates/archivr-core/src/lib.rs \ + crates/archivr-server/src/main.rs \ + crates/archivr-cli/src/main.rs \ + && cargo build --release -p archivr-server -p archivr-cli + +############################################################################### +# Stage 2 – Runtime image +############################################################################### +FROM debian:bookworm-slim + +# Runtime dependencies: +# chromium used by single-file-cli for full-page archiving +# nodejs + npm runtime for single-file-cli +# python3 + pip + venv twitter scraper +# ca-certificates outbound HTTPS from the server +# libssl3 OpenSSL linked by the Rust binary +RUN apt-get update && apt-get install -y --no-install-recommends \ + chromium \ + nodejs \ + npm \ + python3 \ + python3-pip \ + python3-venv \ + ca-certificates \ + libssl3 \ + && rm -rf /var/lib/apt/lists/* + +# Install single-file-cli globally so `single-file` is on PATH. +RUN npm install -g single-file-cli + +# Install yt-dlp and twitter-api-client into an isolated venv to avoid +# conflicts with Debian's system Python packages. +RUN python3 -m venv /opt/archivr-venv \ + && /opt/archivr-venv/bin/pip install --no-cache-dir \ + yt-dlp \ + twitter-api-client + +# Server and CLI binaries (CLI is needed to run `archivr init` on first setup) +COPY --from=builder /build/target/release/archivr-server /usr/local/bin/archivr-server +COPY --from=builder /build/target/release/archivr /usr/local/bin/archivr + +# Pre-built frontend assets (already compiled; no Vite build step needed) +COPY crates/archivr-server/static/ /usr/share/archivr-server/static/ + +# Twitter scraper script +COPY vendor/twitter/scrape_user_tweet_contents.py \ + /usr/local/lib/archivr/scrape_user_tweet_contents.py + +# Wire up env vars that the server (and archivr-core) read at runtime. +# ARCHIVR_BIND and ARCHIVR_TWITTER_CREDENTIALS_FILE are intentionally left +# unset here — set them in docker-compose.yml or at `docker run` time. +ENV ARCHIVR_STATIC_DIR=/usr/share/archivr-server/static \ + ARCHIVR_CHROME=/usr/bin/chromium \ + ARCHIVR_SINGLE_FILE=/usr/local/bin/single-file \ + ARCHIVR_TWEET_PYTHON=/opt/archivr-venv/bin/python3 \ + ARCHIVR_TWEET_SCRAPER=/usr/local/lib/archivr/scrape_user_tweet_contents.py \ + ARCHIVR_YT_DLP=/opt/archivr-venv/bin/yt-dlp + +EXPOSE 8080 + +# Expects the TOML config at /config/archivr-server.toml (mount a volume). +# Copy docker/config.example.toml as a starting point. +ENTRYPOINT ["archivr-server", "/config/archivr-server.toml"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d38ed2a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +services: + archivr: + build: . + image: archivr-server:latest + restart: unless-stopped + ports: + - "8080:8080" + environment: + # Listen on all interfaces inside the container so the port mapping works. + ARCHIVR_BIND: "0.0.0.0:8080" + # Uncomment and set this to enable Twitter/X archiving. + # The file must be accessible inside the container (e.g. in the config volume). + # ARCHIVR_TWITTER_CREDENTIALS_FILE: /config/twitter-cookies.txt + volumes: + # Mount a directory containing archivr-server.toml as read-only config. + # Copy docker/config.example.toml to ./config/archivr-server.toml to start. + - ./config:/config:ro + # Persistent volume for the auth database and archive directories. + # The paths inside must match archive_path values in your TOML config. + - archivr-data:/data + +volumes: + archivr-data: diff --git a/docker/config.example.toml b/docker/config.example.toml new file mode 100644 index 0000000..be582a7 --- /dev/null +++ b/docker/config.example.toml @@ -0,0 +1,32 @@ +# archivr-server.toml — example configuration for Docker deployment. +# +# Copy this file to ./config/archivr-server.toml (next to docker-compose.yml), +# edit it to suit your setup, then run: +# +# docker compose up -d +# +# The bind address defaults to 127.0.0.1:8080. When running under Docker, +# set ARCHIVR_BIND=0.0.0.0:8080 in the environment (docker-compose.yml does +# this already) — the env var takes precedence over the value below. + +# bind = "0.0.0.0:8080" + +# Path to the server-level authentication database. +# Must be on a persistent volume so it survives container restarts. +auth_db_path = "/data/archivr-auth.sqlite" + +# Define one or more archives. +# archive_path must point to the .archivr directory created by `archivr init`, +# on the persistent data volume (/data by default). +# Initialize each archive before starting the server (see Docker quickstart step 2). + +[[archives]] +id = "main" +label = "Main Archive" +archive_path = "/data/archives/main/.archivr" + +# Add more archives as needed: +# [[archives]] +# id = "videos" +# label = "Videos" +# archive_path = "/data/archives/videos/.archivr" diff --git a/docs/README.md b/docs/README.md index 2d018e7..1d796bb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -205,12 +205,17 @@ A `Dockerfile` and `docker-compose.yml` are provided for self-hosting without Ni # edit config/archivr-server.toml — set archive id, label, and archive_path ``` -2. Create the archive directory on the persistent data volume before the first start: +2. Initialize each archive on the persistent data volume before the first start. + The image includes the `archivr` CLI for this purpose: ```sh - docker compose run --rm archivr mkdir -p /data/archives/main + docker compose run --rm archivr archivr init /data/archives/main --name "Main Archive" ``` + This creates `/data/archives/main/.archivr/` with the metadata the server requires. + A bare `mkdir` is not enough — the server reads `name` and `store_path` files that + only `archivr init` writes. + 3. Start the server: ```sh @@ -224,7 +229,12 @@ A `Dockerfile` and `docker-compose.yml` are provided for self-hosting without Ni | Mount | Purpose | |-------|---------| | `./config` (read-only) | Directory containing `archivr-server.toml` | -| `archivr-data` named volume | Auth database and archive directories | +| `archivr-data` named volume | Auth database (`/data/archivr-auth.sqlite`) and archive directories | + +> **Important:** `auth_db_path` must be set explicitly in `archivr-server.toml` to a +> path on the writable data volume (e.g. `/data/archivr-auth.sqlite`). If left unset, +> the server defaults to writing the auth database next to the config file — which is +> on the read-only `/config` mount and will fail. The example config sets this correctly. **Twitter/X archiving** From 93dea9ffbf824fd8b22d6647466eb94d5cb79f8c Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:39:42 +0200 Subject: [PATCH 2/3] fix: address second round of Docker review issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chromium sandbox (P2): - singlefile.rs: add ARCHIVR_CHROME_ARGS env var (space-separated flags appended to Chromium's --browser-args JSON array); Dockerfile sets it to --no-sandbox because Chromium refuses to start as root without it Store-path outside volume (P1): - README: pass explicit absolute store-path as the second positional arg to `archivr init` so the blob store lands on /data instead of the container layer (CLI default is ./.archivr/store, resolved from cwd, which is / with no WORKDIR set) ENTRYPOINT vs CMD (P2): - Dockerfile: switch from ENTRYPOINT to CMD so `docker compose run archivr archivr init …` overrides the full command instead of being appended to the server invocation ffmpeg missing (P2): - Dockerfile: add ffmpeg to the apt-get install block (required by yt-dlp --merge-output-format mp4 for bestvideo+bestaudio streams) Node version (P2): - Dockerfile: replace Debian bookworm's nodejs (18.x) with Node 20 via the NodeSource setup script (single-file-cli declares engines.node >=20) Build context secrets (P2): - Add .dockerignore excluding config/ and docker/ from the build context so runtime secrets (e.g. twitter-cookies.txt) are never sent to the builder - Whitelist .dockerignore in .gitignore docs: - README: document ARCHIVR_CHROME_ARGS in the Environment Variables section --- .dockerignore | 9 ++++++++ .gitignore | 1 + Dockerfile | 21 +++++++++++++------ .../archivr-core/src/downloader/singlefile.rs | 20 ++++++++++++++---- docs/README.md | 7 ++++++- 5 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6c4f628 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +# Exclude runtime config and data directories from the Docker build context. +# The config/ directory may contain secrets (e.g. twitter-cookies.txt) that are +# only needed at runtime via a volume mount — they must never reach the builder. +config/ +docker/ + +# Development and VCS noise +.git/ +.gitignore diff --git a/.gitignore b/.gitignore index 7dbca8c..2f797ce 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ !NEXT.md !Dockerfile +!.dockerignore !docker-compose.yml !docker/ !docker/** diff --git a/Dockerfile b/Dockerfile index 6a451d2..7602011 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,18 +44,24 @@ FROM debian:bookworm-slim # Runtime dependencies: # chromium used by single-file-cli for full-page archiving -# nodejs + npm runtime for single-file-cli +# nodejs (20+) runtime for single-file-cli (requires Node >=20; Debian +# bookworm ships 18, so we install from the NodeSource repo) +# ffmpeg required by yt-dlp to merge separate audio/video streams +# (e.g. YouTube bestvideo+bestaudio format selection) # python3 + pip + venv twitter scraper -# ca-certificates outbound HTTPS from the server +# ca-certificates outbound HTTPS from the server and NodeSource HTTPS # libssl3 OpenSSL linked by the Rust binary RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y --no-install-recommends \ chromium \ nodejs \ - npm \ + ffmpeg \ python3 \ python3-pip \ python3-venv \ - ca-certificates \ libssl3 \ && rm -rf /var/lib/apt/lists/* @@ -88,10 +94,13 @@ ENV ARCHIVR_STATIC_DIR=/usr/share/archivr-server/static \ ARCHIVR_SINGLE_FILE=/usr/local/bin/single-file \ ARCHIVR_TWEET_PYTHON=/opt/archivr-venv/bin/python3 \ ARCHIVR_TWEET_SCRAPER=/usr/local/lib/archivr/scrape_user_tweet_contents.py \ - ARCHIVR_YT_DLP=/opt/archivr-venv/bin/yt-dlp + ARCHIVR_YT_DLP=/opt/archivr-venv/bin/yt-dlp \ + ARCHIVR_CHROME_ARGS=--no-sandbox EXPOSE 8080 # Expects the TOML config at /config/archivr-server.toml (mount a volume). # Copy docker/config.example.toml as a starting point. -ENTRYPOINT ["archivr-server", "/config/archivr-server.toml"] +# Using CMD (not ENTRYPOINT) so `docker compose run archivr archivr init …` +# can override the whole command for first-time archive initialisation. +CMD ["archivr-server", "/config/archivr-server.toml"] diff --git a/crates/archivr-core/src/downloader/singlefile.rs b/crates/archivr-core/src/downloader/singlefile.rs index d50d554..4e19110 100644 --- a/crates/archivr-core/src/downloader/singlefile.rs +++ b/crates/archivr-core/src/downloader/singlefile.rs @@ -68,10 +68,22 @@ fn save_with( // without a writable user-data-dir. Using a subdirectory of temp_dir // keeps it isolated and it gets cleaned up with the rest of the temp dir. let chrome_data_dir = temp_dir.join("chrome-data"); - let browser_args = format!( - "[\"--disable-web-security\",\"--user-data-dir={}\"]", - chrome_data_dir.display() - ); + // Build the browser-args JSON array. Start with the flags always required, + // then append any extra flags from ARCHIVR_CHROME_ARGS (space-separated). + // Docker containers running as root need "--no-sandbox" here because + // Chromium refuses to start as root without it. + let mut chrome_flags = vec![ + "--disable-web-security".to_string(), + format!("--user-data-dir={}", chrome_data_dir.display()), + ]; + if let Ok(extra) = std::env::var("ARCHIVR_CHROME_ARGS") { + chrome_flags.extend(extra.split_whitespace().filter(|s| !s.is_empty()).map(str::to_string)); + } + let quoted: Vec = chrome_flags + .iter() + .map(|f| format!("\"{}\"", f.replace('\\', "\\\\").replace('"', "\\\""))) + .collect(); + let browser_args = format!("[{}]", quoted.join(",")); let out = Command::new(single_file) .arg(url) diff --git a/docs/README.md b/docs/README.md index 1d796bb..e9b616f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -209,7 +209,7 @@ A `Dockerfile` and `docker-compose.yml` are provided for self-hosting without Ni The image includes the `archivr` CLI for this purpose: ```sh - docker compose run --rm archivr archivr init /data/archives/main --name "Main Archive" + docker compose run --rm archivr archivr init /data/archives/main /data/archives/main/.archivr/store --name "Main Archive" ``` This creates `/data/archives/main/.archivr/` with the metadata the server requires. @@ -300,6 +300,11 @@ dependencies (Chromium, Node.js, Python) land in the final layer. - `ARCHIVR_CHROME` - Optional. - Overrides the Chromium/Chrome executable passed to `single-file` via `--browser-executable-path`. Set automatically by the Nix wrapper and the Docker image. Default: `chromium`. +- `ARCHIVR_CHROME_ARGS` + - Optional. + - Space-separated extra flags appended to Chromium's `--browser-args`. The Docker + image sets this to `--no-sandbox` because Chromium refuses to run as root without + it. Leave unset when running natively (Nix, Linux desktop). - `ARCHIVR_TWITTER_CREDENTIALS_FILE` - Required for tweet/thread scraping inputs such as `tweet:ID` and `x:thread:ID`. - Must point to a cookies file for the vendored scraper. From 1e28e1c613d07a8b7266fcd9001e1dfdb44cf4c8 Mon Sep 17 00:00:00 2001 From: TheGeneralist <180094941+thegeneralist01@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:35:08 +0200 Subject: [PATCH 3/3] fix: third round of Docker review issues Rust toolchain (P1): - Dockerfile: bump builder from rust:1.87 to rust:1.88; time@0.3.51, time-core@0.1.9, and time-macros@0.2.30 (present in Cargo.lock) all require MSRV 1.88, so the real cargo build --release step was failing single-file-cli wait mode (P2): - singlefile.rs: replace --browser-wait-until=networkidle2 with networkAlmostIdle; the single-file-cli option only accepts InteractiveTime/networkIdle/networkAlmostIdle/load/domContentLoaded (verified in options.js); networkidle2 is a Puppeteer concept that the CLI does not recognise, causing silent fallback to the earliest state and incomplete captures. networkAlmostIdle is the closest equivalent (<=2 open connections, matching Puppeteer's networkidle2 semantics) Build context size (P3): - .dockerignore: add target/, frontend/node_modules/, frontend/dist/; these can reach 1.4G+ after a local dev build and are never read by the Dockerfile, so sending them to the builder wastes time and memory --- .dockerignore | 6 ++++++ Dockerfile | 2 +- crates/archivr-core/src/downloader/singlefile.rs | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6c4f628..8a8018f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,9 @@ docker/ # Development and VCS noise .git/ .gitignore + +# Generated build outputs — can be 1.4G (target/) and 243M (node_modules/) +# after a local dev build; exclude them to keep the build context small. +target/ +frontend/node_modules/ +frontend/dist/ diff --git a/Dockerfile b/Dockerfile index 7602011..699cd33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ############################################################################### # Stage 1 – Build the Rust server and CLI binaries ############################################################################### -FROM rust:1.87-slim-bookworm AS builder +FROM rust:1.88-slim-bookworm AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ diff --git a/crates/archivr-core/src/downloader/singlefile.rs b/crates/archivr-core/src/downloader/singlefile.rs index 4e19110..97ebb81 100644 --- a/crates/archivr-core/src/downloader/singlefile.rs +++ b/crates/archivr-core/src/downloader/singlefile.rs @@ -90,8 +90,8 @@ fn save_with( .arg(&out_file) .arg(format!("--browser-executable-path={chrome}")) .arg("--browser-headless") - .arg("--browser-wait-until=networkidle2") - // Extra delay after networkidle2: Cloudflare Fonts injects @font-face + .arg("--browser-wait-until=networkAlmostIdle") + // Extra delay after networkAlmostIdle: Cloudflare Fonts injects @font-face // CSS after HTML parse, so the font hook needs more time to see it. .arg("--browser-wait-delay=2000") // Realistic UA: some origins block headless Chrome's default UA string.