diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 8a8018f..0000000 --- a/.dockerignore +++ /dev/null @@ -1,15 +0,0 @@ -# Exclude runtime config and data directories from the Docker build context. -# The config/ directory may contain secrets (e.g. twitter-cookies.txt) that are -# only needed at runtime via a volume mount — they must never reach the builder. -config/ -docker/ - -# Development and VCS noise -.git/ -.gitignore - -# Generated build outputs — can be 1.4G (target/) and 243M (node_modules/) -# after a local dev build; exclude them to keep the build context small. -target/ -frontend/node_modules/ -frontend/dist/ diff --git a/.gitignore b/.gitignore index 2f797ce..754d1d4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,12 +19,6 @@ !ARCHIVR-MENTAL-MODEL.md !NEXT.md -!Dockerfile -!.dockerignore -!docker-compose.yml -!docker/ -!docker/** - !modules/ !modules/** diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 699cd33..0000000 --- a/Dockerfile +++ /dev/null @@ -1,106 +0,0 @@ -# syntax=docker/dockerfile:1 - -############################################################################### -# Stage 1 – Build the Rust server and CLI binaries -############################################################################### -FROM rust:1.88-slim-bookworm AS builder - -RUN apt-get update && apt-get install -y --no-install-recommends \ - pkg-config \ - libssl-dev \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /build - -# Layer the dependency build separately for better cache reuse. -# Stub out every crate so Cargo can resolve and compile all dependencies -# before we copy the real source. -COPY Cargo.toml Cargo.lock ./ -COPY crates/archivr-core/Cargo.toml crates/archivr-core/Cargo.toml -COPY crates/archivr-server/Cargo.toml crates/archivr-server/Cargo.toml -COPY crates/archivr-cli/Cargo.toml crates/archivr-cli/Cargo.toml - -RUN mkdir -p \ - crates/archivr-core/src \ - crates/archivr-server/src \ - crates/archivr-cli/src \ - && touch crates/archivr-core/src/lib.rs \ - && echo 'fn main() {}' > crates/archivr-server/src/main.rs \ - && echo 'fn main() {}' > crates/archivr-cli/src/main.rs \ - && cargo build --release -p archivr-server -p archivr-cli || true - -# Build the real binaries; touch source files to force Cargo to relink. -COPY crates/ crates/ -RUN touch \ - crates/archivr-core/src/lib.rs \ - crates/archivr-server/src/main.rs \ - crates/archivr-cli/src/main.rs \ - && cargo build --release -p archivr-server -p archivr-cli - -############################################################################### -# Stage 2 – Runtime image -############################################################################### -FROM debian:bookworm-slim - -# Runtime dependencies: -# chromium used by single-file-cli for full-page archiving -# nodejs (20+) runtime for single-file-cli (requires Node >=20; Debian -# bookworm ships 18, so we install from the NodeSource repo) -# ffmpeg required by yt-dlp to merge separate audio/video streams -# (e.g. YouTube bestvideo+bestaudio format selection) -# python3 + pip + venv twitter scraper -# ca-certificates outbound HTTPS from the server and NodeSource HTTPS -# libssl3 OpenSSL linked by the Rust binary -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ - && apt-get install -y --no-install-recommends \ - chromium \ - nodejs \ - ffmpeg \ - python3 \ - python3-pip \ - python3-venv \ - libssl3 \ - && rm -rf /var/lib/apt/lists/* - -# Install single-file-cli globally so `single-file` is on PATH. -RUN npm install -g single-file-cli - -# Install yt-dlp and twitter-api-client into an isolated venv to avoid -# conflicts with Debian's system Python packages. -RUN python3 -m venv /opt/archivr-venv \ - && /opt/archivr-venv/bin/pip install --no-cache-dir \ - yt-dlp \ - twitter-api-client - -# Server and CLI binaries (CLI is needed to run `archivr init` on first setup) -COPY --from=builder /build/target/release/archivr-server /usr/local/bin/archivr-server -COPY --from=builder /build/target/release/archivr /usr/local/bin/archivr - -# Pre-built frontend assets (already compiled; no Vite build step needed) -COPY crates/archivr-server/static/ /usr/share/archivr-server/static/ - -# Twitter scraper script -COPY vendor/twitter/scrape_user_tweet_contents.py \ - /usr/local/lib/archivr/scrape_user_tweet_contents.py - -# Wire up env vars that the server (and archivr-core) read at runtime. -# ARCHIVR_BIND and ARCHIVR_TWITTER_CREDENTIALS_FILE are intentionally left -# unset here — set them in docker-compose.yml or at `docker run` time. -ENV ARCHIVR_STATIC_DIR=/usr/share/archivr-server/static \ - ARCHIVR_CHROME=/usr/bin/chromium \ - ARCHIVR_SINGLE_FILE=/usr/local/bin/single-file \ - ARCHIVR_TWEET_PYTHON=/opt/archivr-venv/bin/python3 \ - ARCHIVR_TWEET_SCRAPER=/usr/local/lib/archivr/scrape_user_tweet_contents.py \ - ARCHIVR_YT_DLP=/opt/archivr-venv/bin/yt-dlp \ - ARCHIVR_CHROME_ARGS=--no-sandbox - -EXPOSE 8080 - -# Expects the TOML config at /config/archivr-server.toml (mount a volume). -# Copy docker/config.example.toml as a starting point. -# Using CMD (not ENTRYPOINT) so `docker compose run archivr archivr init …` -# can override the whole command for first-time archive initialisation. -CMD ["archivr-server", "/config/archivr-server.toml"] diff --git a/crates/archivr-core/src/downloader/singlefile.rs b/crates/archivr-core/src/downloader/singlefile.rs index 97ebb81..d50d554 100644 --- a/crates/archivr-core/src/downloader/singlefile.rs +++ b/crates/archivr-core/src/downloader/singlefile.rs @@ -68,30 +68,18 @@ fn save_with( // without a writable user-data-dir. Using a subdirectory of temp_dir // keeps it isolated and it gets cleaned up with the rest of the temp dir. let chrome_data_dir = temp_dir.join("chrome-data"); - // Build the browser-args JSON array. Start with the flags always required, - // then append any extra flags from ARCHIVR_CHROME_ARGS (space-separated). - // Docker containers running as root need "--no-sandbox" here because - // Chromium refuses to start as root without it. - let mut chrome_flags = vec![ - "--disable-web-security".to_string(), - format!("--user-data-dir={}", chrome_data_dir.display()), - ]; - if let Ok(extra) = std::env::var("ARCHIVR_CHROME_ARGS") { - chrome_flags.extend(extra.split_whitespace().filter(|s| !s.is_empty()).map(str::to_string)); - } - let quoted: Vec = chrome_flags - .iter() - .map(|f| format!("\"{}\"", f.replace('\\', "\\\\").replace('"', "\\\""))) - .collect(); - let browser_args = format!("[{}]", quoted.join(",")); + let browser_args = format!( + "[\"--disable-web-security\",\"--user-data-dir={}\"]", + chrome_data_dir.display() + ); let out = Command::new(single_file) .arg(url) .arg(&out_file) .arg(format!("--browser-executable-path={chrome}")) .arg("--browser-headless") - .arg("--browser-wait-until=networkAlmostIdle") - // Extra delay after networkAlmostIdle: Cloudflare Fonts injects @font-face + .arg("--browser-wait-until=networkidle2") + // Extra delay after networkidle2: Cloudflare Fonts injects @font-face // CSS after HTML parse, so the font hook needs more time to see it. .arg("--browser-wait-delay=2000") // Realistic UA: some origins block headless Chrome's default UA string. diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index d38ed2a..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -services: - archivr: - build: . - image: archivr-server:latest - restart: unless-stopped - ports: - - "8080:8080" - environment: - # Listen on all interfaces inside the container so the port mapping works. - ARCHIVR_BIND: "0.0.0.0:8080" - # Uncomment and set this to enable Twitter/X archiving. - # The file must be accessible inside the container (e.g. in the config volume). - # ARCHIVR_TWITTER_CREDENTIALS_FILE: /config/twitter-cookies.txt - volumes: - # Mount a directory containing archivr-server.toml as read-only config. - # Copy docker/config.example.toml to ./config/archivr-server.toml to start. - - ./config:/config:ro - # Persistent volume for the auth database and archive directories. - # The paths inside must match archive_path values in your TOML config. - - archivr-data:/data - -volumes: - archivr-data: diff --git a/docker/config.example.toml b/docker/config.example.toml deleted file mode 100644 index be582a7..0000000 --- a/docker/config.example.toml +++ /dev/null @@ -1,32 +0,0 @@ -# archivr-server.toml — example configuration for Docker deployment. -# -# Copy this file to ./config/archivr-server.toml (next to docker-compose.yml), -# edit it to suit your setup, then run: -# -# docker compose up -d -# -# The bind address defaults to 127.0.0.1:8080. When running under Docker, -# set ARCHIVR_BIND=0.0.0.0:8080 in the environment (docker-compose.yml does -# this already) — the env var takes precedence over the value below. - -# bind = "0.0.0.0:8080" - -# Path to the server-level authentication database. -# Must be on a persistent volume so it survives container restarts. -auth_db_path = "/data/archivr-auth.sqlite" - -# Define one or more archives. -# archive_path must point to the .archivr directory created by `archivr init`, -# on the persistent data volume (/data by default). -# Initialize each archive before starting the server (see Docker quickstart step 2). - -[[archives]] -id = "main" -label = "Main Archive" -archive_path = "/data/archives/main/.archivr" - -# Add more archives as needed: -# [[archives]] -# id = "videos" -# label = "Videos" -# archive_path = "/data/archives/videos/.archivr" diff --git a/docs/README.md b/docs/README.md index e9b616f..2d018e7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -205,17 +205,12 @@ A `Dockerfile` and `docker-compose.yml` are provided for self-hosting without Ni # edit config/archivr-server.toml — set archive id, label, and archive_path ``` -2. Initialize each archive on the persistent data volume before the first start. - The image includes the `archivr` CLI for this purpose: +2. Create the archive directory on the persistent data volume before the first start: ```sh - docker compose run --rm archivr archivr init /data/archives/main /data/archives/main/.archivr/store --name "Main Archive" + docker compose run --rm archivr mkdir -p /data/archives/main ``` - This creates `/data/archives/main/.archivr/` with the metadata the server requires. - A bare `mkdir` is not enough — the server reads `name` and `store_path` files that - only `archivr init` writes. - 3. Start the server: ```sh @@ -229,12 +224,7 @@ A `Dockerfile` and `docker-compose.yml` are provided for self-hosting without Ni | Mount | Purpose | |-------|---------| | `./config` (read-only) | Directory containing `archivr-server.toml` | -| `archivr-data` named volume | Auth database (`/data/archivr-auth.sqlite`) and archive directories | - -> **Important:** `auth_db_path` must be set explicitly in `archivr-server.toml` to a -> path on the writable data volume (e.g. `/data/archivr-auth.sqlite`). If left unset, -> the server defaults to writing the auth database next to the config file — which is -> on the read-only `/config` mount and will fail. The example config sets this correctly. +| `archivr-data` named volume | Auth database and archive directories | **Twitter/X archiving** @@ -300,11 +290,6 @@ dependencies (Chromium, Node.js, Python) land in the final layer. - `ARCHIVR_CHROME` - Optional. - Overrides the Chromium/Chrome executable passed to `single-file` via `--browser-executable-path`. Set automatically by the Nix wrapper and the Docker image. Default: `chromium`. -- `ARCHIVR_CHROME_ARGS` - - Optional. - - Space-separated extra flags appended to Chromium's `--browser-args`. The Docker - image sets this to `--no-sandbox` because Chromium refuses to run as root without - it. Leave unset when running natively (Nix, Linux desktop). - `ARCHIVR_TWITTER_CREDENTIALS_FILE` - Required for tweet/thread scraping inputs such as `tweet:ID` and `x:thread:ID`. - Must point to a cookies file for the vendored scraper.