diff --git a/.gitignore b/.gitignore index bcf6e97..8f160bd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ !docs !docs/** -!src -!src/** +!crates +!crates/** !vendor !vendor/** diff --git a/ARCHIVR-MENTAL-MODEL.md b/ARCHIVR-MENTAL-MODEL.md new file mode 100644 index 0000000..01f4387 --- /dev/null +++ b/ARCHIVR-MENTAL-MODEL.md @@ -0,0 +1,135 @@ +# Archivr Mental Model + +This document explains the current project shape after the workspace refactor. + +## The Big Model + +Archivr is now a Rust workspace with three crates: + +```mermaid +flowchart LR + CLI["archivr-cli"] --> Core["archivr-core"] + Server["archivr-server"] --> Core + UI["static web UI"] --> Server + ServerConfig["server TOML registry"] --> Server + Core --> DB["archive/.archivr/archivr.sqlite"] + Core --> Store["archive store: raw, raw_tweets, structured, temp"] +``` + +The key rule: + +> `archivr-core` owns archive behavior. `archivr-cli` and `archivr-server` are adapters. + +## Crates + +| Crate | Responsibility | +|---|---| +| `archivr-core` | Archive/domain logic, database schema, queries, download/store helpers | +| `archivr-cli` | Command-line interface, argument parsing, terminal behavior | +| `archivr-server` | Web server, API routes, mounted archive registry, static UI | + +## Archive Model + +Each archive is still self-contained: + +```text +some-archive/ + .archivr/ + archivr.sqlite + name + store_path +store/ + raw/ + raw_tweets/ + structured/ + temp/ +``` + +The web server can mount many independent archives through its own TOML registry. +That registry is separate from the archives themselves. + +Example: + +```toml +[[archives]] +id = "personal" +label = "Personal" +archive_path = "/path/to/archive/.archivr" +``` + +## Write Data Flow + +When archiving something through the CLI: + +```mermaid +sequenceDiagram + participant User + participant CLI + participant Core + participant Store + participant DB + + User->>CLI: archivr archive path-or-url + CLI->>Core: classify source and call downloader/store helpers + Core->>Store: save raw/structured artifacts + Core->>DB: insert run, source identity, entry, artifacts + CLI->>User: terminal result +``` + +## Read Data Flow + +When opening the web UI: + +```mermaid +sequenceDiagram + participant Browser + participant Server + participant Core + participant DB + + Browser->>Server: GET /api/archives + Browser->>Server: GET /api/archives/:id/entries + Server->>Core: list_root_entries(conn) + Core->>DB: query archive SQLite + DB-->>Core: rows + Core-->>Server: summaries + Server-->>Browser: JSON +``` + +## Where To Edit + +| Feature kind | Edit here | +|---|---| +| DB schema, inserts, archive runs, entries, tags | `crates/archivr-core/src/database.rs` | +| Archive opening, listing entries, entry detail, runs | `crates/archivr-core/src/archive.rs` | +| Download/save behavior | `crates/archivr-core/src/downloader/` | +| CLI commands, argument parsing, terminal output | `crates/archivr-cli/src/main.rs` | +| Server API routes | `crates/archivr-server/src/routes.rs` | +| Mounted archive config model | `crates/archivr-server/src/registry.rs` | +| Browser UI behavior | `crates/archivr-server/static/app.js` | +| Browser UI layout | `crates/archivr-server/static/index.html` | +| Browser UI styling | `crates/archivr-server/static/styles.css` | + +## Practical Feature Rule + +If a feature affects archive truth, start in `archivr-core`. + +If a feature is only how the terminal behaves, edit `archivr-cli`. + +If a feature is only how the browser sees or calls things, edit `archivr-server` and the static UI. + +If a browser feature needs new data, the usual order is: + +1. Add or query the data in `archivr-core`. +2. Expose it in `archivr-server`. +3. Render it in the static UI. + +## Current Limitations + +The web server reads archive data and serves the UI. It does not yet implement capture. + +Search is currently simple client-side filtering. + +Auth is not a production model yet. + +Admin is a mounted-archives view, not a management system. diff --git a/Cargo.lock b/Cargo.lock index 155a9fc..5a6b629 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -56,7 +68,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -67,7 +79,7 @@ checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -77,24 +89,132 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] -name = "archivr" +name = "archivr-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "archivr-core", + "chrono", + "clap", + "regex", + "rusqlite", + "serde_json", +] + +[[package]] +name = "archivr-core" version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap", "hex", "regex", + "rusqlite", + "serde", + "serde_json", "sha3", "uuid", ] +[[package]] +name = "archivr-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "archivr-core", + "axum", + "serde", + "tempfile", + "tokio", + "toml", + "tower", + "tower-http", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + [[package]] name = "block-buffer" version = "0.10.4" @@ -110,6 +230,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "cc" version = "1.2.41" @@ -220,12 +346,94 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -245,7 +453,31 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi", + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", ] [[package]] @@ -260,6 +492,92 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -284,12 +602,28 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "js-sys" version = "0.3.81" @@ -311,9 +645,26 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "log" @@ -321,12 +672,45 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -348,6 +732,24 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "proc-macro2" version = "1.0.101" @@ -401,12 +803,120 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.60.2", +] + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha3" version = "0.10.8" @@ -423,6 +933,28 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "strsim" version = "0.11.1" @@ -440,12 +972,191 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.60.2", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.19" @@ -469,12 +1180,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.14.7+wasi-0.2.4" @@ -620,6 +1343,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.53.5" @@ -685,8 +1417,43 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index f40ba88..62f4b6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,29 @@ -[package] -name = "archivr" +[workspace] +members = [ + "crates/archivr-core", + "crates/archivr-cli", + "crates/archivr-server", +] +resolver = "2" + +[workspace.package] version = "0.1.0" edition = "2024" -[dependencies] +[workspace.dependencies] anyhow = "1.0.100" +axum = "0.7.9" chrono = "0.4.42" clap = { version = "4.5.48", features = ["derive"] } hex = "0.4.3" regex = "1.12.2" +rusqlite = { version = "0.32.1", features = ["bundled"] } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.132" sha3 = "0.10.8" +tempfile = "3.13.0" +tokio = { version = "1.41.1", features = ["macros", "rt-multi-thread", "net"] } +toml = "0.8.19" +tower = "0.5.1" +tower-http = { version = "0.6.2", features = ["fs", "trace"] } uuid = { version = "1.18.1", features = ["v4"] } diff --git a/crates/archivr-cli/Cargo.toml b/crates/archivr-cli/Cargo.toml new file mode 100644 index 0000000..3a190d7 --- /dev/null +++ b/crates/archivr-cli/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "archivr-cli" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "archivr" +path = "src/main.rs" + +[dependencies] +archivr-core = { path = "../archivr-core" } +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +regex.workspace = true +rusqlite.workspace = true +serde_json.workspace = true diff --git a/src/main.rs b/crates/archivr-cli/src/main.rs similarity index 57% rename from src/main.rs rename to crates/archivr-cli/src/main.rs index 177194b..d47c9c9 100644 --- a/src/main.rs +++ b/crates/archivr-cli/src/main.rs @@ -1,16 +1,15 @@ -use anyhow::Result; +use anyhow::{Context, Result}; +use archivr_core::{archive, database, downloader, twitter::parse_tweet_id}; use chrono::Local; use clap::{Parser, Subcommand}; +use serde_json::json; use std::{ + collections::HashSet, env, fs, path::{Path, PathBuf}, process, }; -mod downloader; -mod hash; -mod twitter; - #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { @@ -54,19 +53,6 @@ enum Command { }, } -fn get_archive_path() -> Option { - let mut dir = env::current_dir().unwrap(); - loop { - if dir.join(".archivr").is_dir() { - return Some(dir.join(".archivr")); - } - if !dir.pop() { - break; - } - } - None -} - #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum Source { YouTubeVideo, @@ -84,17 +70,11 @@ enum Source { Other, } -use crate::twitter::parse_tweet_id; - fn expand_shorthand_to_url(path: &str, source: &Source) -> String { if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) { - return format!( - "https://x.com/i/status/{}", - path.split(':') - .next_back() - .and_then(parse_tweet_id) - .unwrap() - ); + if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) { + return format!("https://x.com/i/status/{tweet_id}"); + } } if let Some(path) = path.strip_prefix("instagram:") { @@ -221,7 +201,8 @@ fn determine_source(path: &str) -> Source { return Source::Local; } else if path.starts_with("http://") || path.starts_with("https://") { // Video URLs (watch, youtu.be, shorts) - let video_re = regex::Regex::new(r"^https?://(?:www\.)?(?:youtu\.be/[0-9A-Za-z_-]+|youtube\.com/watch\?v=[0-9A-Za-z_-]+|youtube\.com/shorts/[0-9A-Za-z_-]+)").unwrap(); + let video_re = regex::Regex::new(r"^https?://(?:www\.)?(?:youtu\.be/[0-9A-Za-z_-]+|youtube\.com/watch\?v=[0-9A-Za-z_-]+|youtube\.com/shorts/[0-9A-Za-z_-]+)") + .expect("YouTube video URL regex literal must be valid"); if video_re.is_match(path) { return Source::YouTubeVideo; } @@ -229,13 +210,14 @@ fn determine_source(path: &str) -> Source { // Playlist URLs let playlist_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/playlist\?list=[0-9A-Za-z_-]+") - .unwrap(); + .expect("YouTube playlist URL regex literal must be valid"); if playlist_re.is_match(path) { return Source::YouTubePlaylist; } // Channel or user URLs (channel IDs, /c/, /user/, or @handles) - let channel_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/(?:channel/[0-9A-Za-z_-]+|c/[0-9A-Za-z_-]+|user/[0-9A-Za-z_-]+|@[0-9A-Za-z_-]+)").unwrap(); + let channel_re = regex::Regex::new(r"^https?://(?:www\.)?youtube\.com/(?:channel/[0-9A-Za-z_-]+|c/[0-9A-Za-z_-]+|user/[0-9A-Za-z_-]+|@[0-9A-Za-z_-]+)") + .expect("YouTube channel URL regex literal must be valid"); if channel_re.is_match(path) { return Source::YouTubeChannel; } @@ -288,73 +270,328 @@ fn determine_source(path: &str) -> Source { return Source::Snapchat; } } + if Path::new(path).exists() { + return Source::Local; + } Source::Other } -fn hash_exists(filename: String, store_path: &Path) -> bool { - let mut chars = filename.chars(); - let first_letter = chars.next().unwrap(); - let second_letter = chars.next().unwrap(); - - let path = store_path - .join("raw") - .join(first_letter.to_string()) - .join(second_letter.to_string()) - .join(filename); +fn hash_exists(hash: &str, file_extension: &str, store_path: &Path) -> Result { + let path = store_path.join(raw_relative_path_from_hash(hash, file_extension)?); println!("Checking {}", path.display()); - path.exists() + Ok(path.exists()) } -fn move_temp_to_raw(file: &Path, hash: &String, store_path: &Path) -> Result<()> { - let mut chars = hash.chars(); - let first_letter = chars.next().unwrap().to_string(); - let second_letter = chars.next().unwrap().to_string(); +fn move_temp_to_raw(file: &Path, hash: &str, store_path: &Path) -> Result<()> { let file_extension = file .extension() .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())); + let raw_relpath = raw_relative_path_from_hash(hash, &file_extension)?; + let destination = store_path.join(raw_relpath); - fs::create_dir_all( - store_path - .join("raw") - .join(&first_letter) - .join(&second_letter), - )?; + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent)?; + } - fs::rename( - file, - store_path - .join("raw") - .join(&first_letter) - .join(&second_letter) - .join(format!( - "{hash}{}", - if file_extension.is_empty() { - "" - } else { - &file_extension - } - )), - )?; + fs::rename(file, destination)?; Ok(()) } -fn initialize_store_directories(store_path: &Path) -> Result<()> { - fs::create_dir_all(store_path.join("raw"))?; - fs::create_dir_all(store_path.join("raw_tweets"))?; - fs::create_dir_all(store_path.join("structured"))?; - fs::create_dir_all(store_path.join("temp"))?; +fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result { + let mut chars = hash.chars(); + let first_letter = chars.next().context("hash must not be empty")?; + let second_letter = chars + .next() + .context("hash must be at least two characters")?; + + Ok(PathBuf::from("raw") + .join(first_letter.to_string()) + .join(second_letter.to_string()) + .join(format!("{hash}{file_extension}"))) +} + +fn path_to_store_string(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +fn extension_without_dot(file_extension: &str) -> Option { + file_extension + .strip_prefix('.') + .filter(|extension| !extension.is_empty()) + .map(|extension| extension.to_string()) +} + +fn blob_record_for_raw_relpath( + store_path: &Path, + raw_relpath: &Path, +) -> Result { + let absolute_path = store_path.join(raw_relpath); + let file_name = raw_relpath + .file_name() + .and_then(|name| name.to_str()) + .context("raw artifact path must have a UTF-8 file name")?; + let (sha256, extension) = match file_name.rsplit_once('.') { + Some((hash, extension)) => (hash.to_string(), Some(extension.to_string())), + None => (file_name.to_string(), None), + }; + + Ok(database::BlobRecord { + sha256, + byte_size: fs::metadata(&absolute_path) + .with_context(|| format!("failed to stat raw artifact {}", absolute_path.display()))? + .len() as i64, + mime_type: None, + extension, + raw_relpath: path_to_store_string(raw_relpath), + }) +} + +fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str) { + match source { + Source::YouTubeVideo => ("youtube", "video", "video"), + Source::YouTubePlaylist => ("youtube", "playlist", "container"), + Source::YouTubeChannel => ("youtube", "channel", "container"), + Source::X => ("x", "post", "video"), + Source::Tweet => ("x", "tweet", "tweet_json"), + Source::TweetThread => ("x", "tweet_thread", "tweet_json"), + Source::Instagram => ("instagram", "post", "video"), + Source::Facebook => ("facebook", "post", "video"), + Source::TikTok => ("tiktok", "video", "video"), + Source::Reddit => ("reddit", "post", "video"), + Source::Snapchat => ("snapchat", "story", "video"), + Source::Local => ("local", "file", "file"), + Source::Other => ("other", "unknown", "unknown"), + } +} + +fn local_file_extension(path: &str) -> String { + Path::new(path.trim_start_matches("file://")) + .extension() + .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) +} + +fn media_file_extension(source: Source, path: &str) -> String { + match source { + Source::YouTubeVideo + | Source::X + | Source::Instagram + | Source::Facebook + | Source::TikTok + | Source::Reddit + | Source::Snapchat => ".mp4".to_string(), + Source::Local => local_file_extension(path), + _ => String::new(), + } +} + +fn tweet_id_from_archive_path(path: &str) -> Option { + path.split(':').next_back().and_then(parse_tweet_id) +} + +fn create_structured_root(store_path: &Path, entry: &database::ArchivedEntry) -> Result<()> { + debug_assert!(entry.entry_uid.starts_with("entry_")); + fs::create_dir_all(store_path.join(&entry.structured_root_relpath))?; Ok(()) } +fn record_media_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + canonical_locator: &str, + source: Source, + hash: &str, + file_extension: &str, + byte_size: i64, +) -> Result { + debug_assert!(run.run_uid.starts_with("run_")); + debug_assert!(item.item_uid.starts_with("item_")); + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let raw_relpath = raw_relative_path_from_hash(hash, file_extension)?; + let blob = database::BlobRecord { + sha256: hash.to_string(), + byte_size, + mime_type: None, + extension: extension_without_dot(file_extension), + raw_relpath: path_to_store_string(&raw_relpath), + }; + let blob_id = database::upsert_blob(conn, &blob)?; + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + None, + Some(canonical_locator), + canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "requested_locator": requested_locator, + "canonical_locator": canonical_locator + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "primary_media".to_string(), + storage_area: "raw".to_string(), + relpath: blob.raw_relpath, + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + )?; + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + +fn record_tweet_entry( + conn: &rusqlite::Connection, + store_path: &Path, + user_id: i64, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + requested_locator: &str, + source: Source, + tweet_id: &str, +) -> Result { + debug_assert!(run.run_uid.starts_with("run_")); + debug_assert!(item.item_uid.starts_with("item_")); + let (source_kind, entity_kind, representation_kind) = source_metadata(source); + let canonical_locator = format!("https://x.com/i/status/{tweet_id}"); + let source_identity_id = database::upsert_source_identity( + conn, + source_kind, + entity_kind, + Some(tweet_id), + Some(&canonical_locator), + &canonical_locator, + )?; + let entry = database::create_archived_entry( + conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: source_kind.to_string(), + entity_kind: entity_kind.to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: representation_kind.to_string(), + source_metadata_json: json!({ + "tweet_id": tweet_id, + "requested_locator": requested_locator + }) + .to_string(), + display_metadata_json: None, + }, + )?; + create_structured_root(store_path, &entry)?; + + let tweet_json_relpath = PathBuf::from("raw_tweets").join(format!("tweet-{tweet_id}.json")); + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "raw_tweet_json".to_string(), + storage_area: "raw_tweets".to_string(), + relpath: path_to_store_string(&tweet_json_relpath), + blob_id: None, + logical_path: None, + metadata_json: None, + }, + )?; + + let tweet_json = fs::read_to_string(store_path.join(&tweet_json_relpath))?; + for (role, raw_relpath) in tweet_raw_artifacts(&tweet_json)? { + let raw_path = PathBuf::from(&raw_relpath); + let blob = blob_record_for_raw_relpath(store_path, &raw_path)?; + let blob_id = database::upsert_blob(conn, &blob)?; + database::add_entry_artifact( + conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: role, + storage_area: "raw".to_string(), + relpath: raw_relpath, + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + )?; + } + + database::complete_archive_run_item(conn, item.id, entry.id)?; + Ok(entry) +} + +fn tweet_raw_artifacts(tweet_json: &str) -> Result> { + let regex = regex::Regex::new(r#""(avatar_local_path|local_path)": "([^"\n]+)""#)?; + let mut seen = HashSet::new(); + let mut artifacts = Vec::new(); + + for captures in regex.captures_iter(tweet_json) { + let relpath = captures[2].to_string(); + if !relpath.starts_with("raw/") || !seen.insert(relpath.clone()) { + continue; + } + + let role = if &captures[1] == "avatar_local_path" { + "avatar" + } else { + "media" + }; + artifacts.push((role.to_string(), relpath)); + } + + Ok(artifacts) +} + +fn fail_archive_and_exit( + conn: &rusqlite::Connection, + run: &database::ArchiveRun, + item: &database::ArchiveRunItem, + message: &str, +) -> ! { + let _ = database::fail_archive_run_item(conn, item.id, message); + let _ = database::fail_archive_run(conn, run.id, message); + eprintln!("{message}"); + process::exit(1); +} + fn main() -> Result<()> { let args = Args::parse(); match args.command { Command::Archive { ref path } => { - let archive_path = match get_archive_path() { + let archive_path = match archive::find_archive_path()? { Some(path) => path, None => { eprintln!("Not in an archive. Use 'archivr init' to create one."); @@ -375,14 +612,42 @@ fn main() -> Result<()> { }; let source = determine_source(path); + let (source_kind, entity_kind, _) = source_metadata(source); + let conn = database::open_or_initialize(&archive_path)?; + let user_id = database::ensure_default_user(&conn)?; + let run = database::create_archive_run(&conn, user_id, 1)?; + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + path, + None, + source_kind, + entity_kind, + )?; // Sources: Tweets or Twitter Threads match source { Source::Other => { - eprintln!("Archiving from this source is not yet implemented."); - process::exit(1); + fail_archive_and_exit( + &conn, + &run, + &item, + "Archiving from this source is not yet implemented.", + ); } Source::Tweet | Source::TweetThread => { + let tweet_id = match tweet_id_from_archive_path(path) { + Some(tweet_id) => tweet_id, + None => fail_archive_and_exit( + &conn, + &run, + &item, + "Failed to archive tweet: invalid tweet ID", + ), + }; + match downloader::tweets::archive( path, source == Source::TweetThread, @@ -390,6 +655,17 @@ fn main() -> Result<()> { ×tamp, ) { Ok(true) => { + record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + path, + source, + &tweet_id, + )?; + database::finish_archive_run(&conn, run.id)?; println!( "Tweet archived successfully to {}", store_path.join("raw_tweets").display() @@ -397,6 +673,17 @@ fn main() -> Result<()> { return Ok(()); } Ok(false) => { + record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + path, + source, + &tweet_id, + )?; + database::finish_archive_run(&conn, run.id)?; println!( "Tweet already archived in {}", store_path.join("raw_tweets").display() @@ -404,8 +691,12 @@ fn main() -> Result<()> { return Ok(()); } Err(e) => { - eprintln!("Failed to archive tweet: {e}"); - process::exit(1); + fail_archive_and_exit( + &conn, + &run, + &item, + &format!("Failed to archive tweet: {e}"), + ); } } } @@ -413,6 +704,7 @@ fn main() -> Result<()> { } // Sources, for which yt-dlp is needed + let requested_path = path.to_string(); let path = expand_shorthand_to_url(path, &source); let hash = match source { Source::YouTubeVideo @@ -425,8 +717,12 @@ fn main() -> Result<()> { match downloader::ytdlp::download(path.clone(), &store_path, ×tamp) { Ok(h) => h, Err(e) => { - eprintln!("Failed to download from YouTube: {e}"); - process::exit(1); + fail_archive_and_exit( + &conn, + &run, + &item, + &format!("Failed to download media: {e}"), + ); } } } @@ -434,31 +730,36 @@ fn main() -> Result<()> { match downloader::local::save(path.clone(), &store_path, ×tamp) { Ok(h) => h, Err(e) => { - eprintln!("Failed to archive local file: {e}"); - process::exit(1); + fail_archive_and_exit( + &conn, + &run, + &item, + &format!("Failed to archive local file: {e}"), + ); } } } + Source::YouTubePlaylist | Source::YouTubeChannel => { + fail_archive_and_exit( + &conn, + &run, + &item, + "Playlist and channel container expansion are not yet implemented.", + ); + } _ => unreachable!(), }; - let file_extension = match source { - Source::YouTubeVideo - | Source::X - | Source::Instagram - | Source::Facebook - | Source::TikTok - | Source::Reddit - | Source::Snapchat => ".mp4", - Source::Local => { - let p = Path::new(path.trim_start_matches("file://")); - &p.extension() - .map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy())) - } - _ => "", - }; + let file_extension = media_file_extension(source, &path); + let temp_file = store_path + .join("temp") + .join(×tamp) + .join(format!("{timestamp}{file_extension}")); + let byte_size = fs::metadata(&temp_file) + .with_context(|| format!("failed to stat staged file {}", temp_file.display()))? + .len() as i64; - let hash_exists = hash_exists(format!("{hash}{file_extension}"), &store_path); + let hash_exists = hash_exists(&hash, &file_extension, &store_path)?; // TODO: check for repeated archives? // There could be one of the following: @@ -490,9 +791,20 @@ fn main() -> Result<()> { println!("File archived successfully."); } - // TODO: DB INSERT, inserting a record - // https://github.com/rusqlite/rusqlite - // Think of the DB schema + record_media_entry( + &conn, + &store_path, + user_id, + &run, + &item, + &requested_path, + &path, + source, + &hash, + &file_extension, + byte_size, + )?; + database::finish_archive_run(&conn, run.id)?; Ok(()) } @@ -503,48 +815,26 @@ fn main() -> Result<()> { name: ref archive_name, force_with_info_removal, } => { - let archive_path = Path::new(&archive_path_string).join(".archivr"); + let archive_parent = Path::new(&archive_path_string); let store_path = if Path::new(&store_path_string).is_relative() { - env::current_dir().unwrap().join(store_path_string) + env::current_dir() + .context("failed to read current working directory")? + .join(store_path_string) } else { Path::new(store_path_string).to_path_buf() }; - if archive_path.exists() { - if !archive_path.is_dir() { - eprintln!( - "Archive path exists and is not a directory: {}", - archive_path.display() - ); - process::exit(1); - } + let paths = archive::initialize_archive( + archive_parent, + &store_path, + archive_name, + force_with_info_removal, + )?; - if force_with_info_removal { - fs::remove_dir_all(&archive_path)?; - } else if fs::read_dir(&archive_path)?.next().is_some() { - eprintln!( - "Archive already exists at {} and is not empty. Use --force-with-info-removal to reinitialize.", - archive_path.display() - ); - process::exit(1); - } - } - - if store_path.exists() && !force_with_info_removal { - eprintln!("Store path already exists at {}", store_path.display()); - process::exit(1); - } - - fs::create_dir_all(&archive_path).unwrap(); - fs::create_dir_all(&store_path).unwrap(); - fs::write(archive_path.join("name"), archive_name).unwrap(); - let _ = fs::write( - archive_path.join("store_path"), - store_path.canonicalize().unwrap().to_str().unwrap(), + println!( + "Initialized empty archive in {}", + paths.archive_path.display() ); - initialize_store_directories(&store_path).unwrap(); - - println!("Initialized empty archive in {}", archive_path.display()); Ok(()) } // _ => eprintln!("Unknown command: {:?}", args.command), @@ -909,6 +1199,16 @@ mod tests { } } + #[test] + fn test_existing_local_path_source() { + let path = env::current_dir().unwrap().join("Cargo.toml"); + assert_eq!( + determine_source(path.to_str().unwrap()), + Source::Local, + "existing filesystem paths should be archived as local files" + ); + } + #[test] fn test_initialize_store_directories() { let store_path = env::temp_dir().join(format!( @@ -916,7 +1216,7 @@ mod tests { Local::now().format("%Y%m%d%H%M%S%3f") )); - initialize_store_directories(&store_path).unwrap(); + archive::initialize_store_directories(&store_path).unwrap(); assert!(store_path.join("raw").is_dir()); assert!(store_path.join("raw_tweets").is_dir()); @@ -926,4 +1226,96 @@ mod tests { fs::remove_dir_all(store_path).unwrap(); } + + #[test] + fn test_record_tweet_entry_links_json_and_raw_artifacts() { + let store_path = env::temp_dir().join(format!( + "archivr-tweet-db-test-{}", + Local::now().format("%Y%m%d%H%M%S%3f") + )); + let _ = fs::remove_dir_all(&store_path); + archive::initialize_store_directories(&store_path).unwrap(); + fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap(); + fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap(); + fs::write( + store_path + .join("raw") + .join("a") + .join("b") + .join("abcdef.jpg"), + b"avatar", + ) + .unwrap(); + fs::write( + store_path + .join("raw") + .join("c") + .join("d") + .join("cdef01.mp4"), + b"media", + ) + .unwrap(); + fs::write( + store_path.join("raw_tweets").join("tweet-123.json"), + r#"{ + "author": { "avatar_local_path": "raw/a/b/abcdef.jpg" }, + "entities": { "media": [{ "local_path": "raw/c/d/cdef01.mp4" }] } +}"#, + ) + .unwrap(); + + let conn = rusqlite::Connection::open_in_memory().unwrap(); + database::initialize_schema(&conn).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 1).unwrap(); + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + "tweet:123", + None, + "x", + "tweet", + ) + .unwrap(); + + let entry = record_tweet_entry( + &conn, + &store_path, + user_id, + &run, + &item, + "tweet:123", + Source::Tweet, + "123", + ) + .unwrap(); + database::finish_archive_run(&conn, run.id).unwrap(); + + let artifact_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM entry_artifacts WHERE entry_id = ?1", + [entry.id], + |row| row.get(0), + ) + .unwrap(); + let blob_count: i64 = conn + .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get(0)) + .unwrap(); + let run_status: String = conn + .query_row( + "SELECT status FROM archive_runs WHERE id = ?1", + [run.id], + |row| row.get(0), + ) + .unwrap(); + + assert_eq!(artifact_count, 3); + assert_eq!(blob_count, 2); + assert_eq!(run_status, "completed"); + assert!(store_path.join(&entry.structured_root_relpath).is_dir()); + + let _ = fs::remove_dir_all(store_path); + } } diff --git a/crates/archivr-core/Cargo.toml b/crates/archivr-core/Cargo.toml new file mode 100644 index 0000000..2325fdf --- /dev/null +++ b/crates/archivr-core/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "archivr-core" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +hex.workspace = true +regex.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +sha3.workspace = true +uuid.workspace = true diff --git a/crates/archivr-core/src/archive.rs b/crates/archivr-core/src/archive.rs new file mode 100644 index 0000000..0d507c0 --- /dev/null +++ b/crates/archivr-core/src/archive.rs @@ -0,0 +1,438 @@ +use anyhow::{Context, Result, bail}; +use rusqlite::OptionalExtension; +use std::{ + env, fs, + path::{Path, PathBuf}, +}; + +use crate::database; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchivePaths { + pub archive_path: PathBuf, + pub store_path: PathBuf, + pub name: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct EntrySummary { + pub entry_uid: String, + pub archived_at: String, + pub source_kind: String, + pub entity_kind: String, + pub title: Option, + pub visibility: String, + pub original_url: Option, + pub artifact_count: i64, + pub total_artifact_bytes: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct EntryDetail { + pub summary: EntrySummary, + pub structured_root_relpath: String, + pub source_metadata_json: String, + pub display_metadata_json: Option, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct EntryArtifactSummary { + pub artifact_role: String, + pub storage_area: String, + pub relpath: String, + pub byte_size: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct RunSummary { + pub run_uid: String, + pub started_at: String, + pub finished_at: Option, + pub status: String, + pub requested_count: i64, + pub discovered_count: i64, + pub completed_count: i64, + pub failed_count: i64, + pub error_summary: Option, +} + +pub fn find_archive_path_from(start: &Path) -> Result> { + let mut dir = start.to_path_buf(); + loop { + let candidate = dir.join(".archivr"); + if candidate.is_dir() { + return Ok(Some(candidate)); + } + if !dir.pop() { + return Ok(None); + } + } +} + +pub fn find_archive_path() -> Result> { + let cwd = env::current_dir().context("failed to read current working directory")?; + find_archive_path_from(&cwd) +} + +pub fn read_archive_paths(archive_path: &Path) -> Result { + if !archive_path.is_dir() { + bail!("archive path does not exist: {}", archive_path.display()); + } + + let name = fs::read_to_string(archive_path.join("name")) + .with_context(|| format!("failed to read archive name in {}", archive_path.display()))? + .trim() + .to_string(); + let store_path = fs::read_to_string(archive_path.join("store_path")) + .with_context(|| format!("failed to read store path in {}", archive_path.display()))?; + + Ok(ArchivePaths { + archive_path: archive_path.to_path_buf(), + store_path: PathBuf::from(store_path.trim()), + name, + }) +} + +pub fn initialize_archive( + archive_parent: &Path, + store_path: &Path, + archive_name: &str, + force_with_info_removal: bool, +) -> Result { + let archive_path = archive_parent.join(".archivr"); + + if archive_path.exists() { + if !archive_path.is_dir() { + bail!( + "Archive path exists and is not a directory: {}", + archive_path.display() + ); + } + + if force_with_info_removal { + fs::remove_dir_all(&archive_path)?; + } else if fs::read_dir(&archive_path)?.next().is_some() { + bail!( + "Archive already exists at {} and is not empty. Use --force-with-info-removal to reinitialize.", + archive_path.display() + ); + } + } + + if store_path.exists() && !force_with_info_removal { + bail!("Store path already exists at {}", store_path.display()); + } + + fs::create_dir_all(&archive_path)?; + fs::create_dir_all(store_path)?; + fs::write(archive_path.join("name"), archive_name)?; + let canonical_store_path = store_path + .canonicalize() + .with_context(|| format!("failed to canonicalize {}", store_path.display()))?; + fs::write( + archive_path.join("store_path"), + canonical_store_path + .to_str() + .context("store path is not valid UTF-8")?, + )?; + + initialize_store_directories(&canonical_store_path)?; + let conn = database::open_or_initialize(&archive_path)?; + let _ = database::ensure_default_user(&conn)?; + + Ok(ArchivePaths { + archive_path, + store_path: canonical_store_path, + name: archive_name.to_string(), + }) +} + +pub fn initialize_store_directories(store_path: &Path) -> Result<()> { + fs::create_dir_all(store_path.join("raw"))?; + fs::create_dir_all(store_path.join("raw_tweets"))?; + fs::create_dir_all(store_path.join("structured"))?; + fs::create_dir_all(store_path.join("temp"))?; + Ok(()) +} + +pub fn list_root_entries(conn: &rusqlite::Connection) -> Result> { + let mut stmt = conn.prepare( + "SELECT + e.entry_uid, + e.archived_at, + e.source_kind, + e.entity_kind, + e.title, + e.visibility, + si.canonical_url, + COUNT(ea.id) AS artifact_count, + COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes + FROM archived_entries e + JOIN source_identities si ON si.id = e.source_identity_id + LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id + LEFT JOIN blobs b ON b.id = ea.blob_id + WHERE e.parent_entry_id IS NULL + GROUP BY e.id + ORDER BY e.archived_at DESC, e.id DESC", + )?; + + let entries = stmt + .query_map([], |row| { + Ok(EntrySummary { + entry_uid: row.get(0)?, + archived_at: row.get(1)?, + source_kind: row.get(2)?, + entity_kind: row.get(3)?, + title: row.get(4)?, + visibility: row.get(5)?, + original_url: row.get(6)?, + artifact_count: row.get(7)?, + total_artifact_bytes: row.get(8)?, + }) + })? + .collect::>>()?; + + Ok(entries) +} + +pub fn get_entry_detail( + conn: &rusqlite::Connection, + entry_uid: &str, +) -> Result> { + let Some((entry_id, structured_root_relpath, source_metadata_json, display_metadata_json)) = + conn.query_row( + "SELECT id, structured_root_relpath, source_metadata_json, display_metadata_json + FROM archived_entries + WHERE entry_uid = ?1", + [entry_uid], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + }, + ) + .optional()? + else { + return Ok(None); + }; + + let summary = list_root_entries(conn)? + .into_iter() + .find(|entry| entry.entry_uid == entry_uid) + .context("entry disappeared while loading detail")?; + + let mut stmt = conn.prepare( + "SELECT ea.artifact_role, ea.storage_area, ea.relpath, b.byte_size + FROM entry_artifacts ea + LEFT JOIN blobs b ON b.id = ea.blob_id + WHERE ea.entry_id = ?1 + ORDER BY ea.id ASC", + )?; + let artifacts = stmt + .query_map([entry_id], |row| { + Ok(EntryArtifactSummary { + artifact_role: row.get(0)?, + storage_area: row.get(1)?, + relpath: row.get(2)?, + byte_size: row.get(3)?, + }) + })? + .collect::>>()?; + + Ok(Some(EntryDetail { + summary, + structured_root_relpath, + source_metadata_json, + display_metadata_json, + artifacts, + })) +} + +pub fn list_runs(conn: &rusqlite::Connection) -> Result> { + let mut stmt = conn.prepare( + "SELECT run_uid, started_at, finished_at, status, requested_count, + discovered_count, completed_count, failed_count, error_summary + FROM archive_runs + ORDER BY started_at DESC, id DESC", + )?; + let runs = stmt + .query_map([], |row| { + Ok(RunSummary { + run_uid: row.get(0)?, + started_at: row.get(1)?, + finished_at: row.get(2)?, + status: row.get(3)?, + requested_count: row.get(4)?, + discovered_count: row.get(5)?, + completed_count: row.get(6)?, + failed_count: row.get(7)?, + error_summary: row.get(8)?, + }) + })? + .collect::>>()?; + + Ok(runs) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn unique_path(prefix: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + env::temp_dir().join(format!("{prefix}-{nanos}-{}", std::process::id())) + } + + #[test] + fn find_archive_path_walks_up_to_dot_archivr() { + let root = unique_path("archivr-core-find"); + let nested = root.join("a").join("b"); + fs::create_dir_all(root.join(".archivr")).unwrap(); + fs::create_dir_all(&nested).unwrap(); + + let found = find_archive_path_from(&nested).unwrap().unwrap(); + + assert_eq!(found, root.join(".archivr")); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn read_archive_paths_returns_name_and_store_path() { + let root = unique_path("archivr-core-open"); + let archive_path = root.join(".archivr"); + let store_path = root.join("store"); + fs::create_dir_all(&archive_path).unwrap(); + fs::create_dir_all(&store_path).unwrap(); + fs::write(archive_path.join("name"), "Personal").unwrap(); + fs::write( + archive_path.join("store_path"), + store_path.display().to_string(), + ) + .unwrap(); + + let paths = read_archive_paths(&archive_path).unwrap(); + + assert_eq!(paths.archive_path, archive_path); + assert_eq!(paths.store_path, store_path); + assert_eq!(paths.name, "Personal"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn initialize_archive_creates_database_store_and_metadata() { + let root = unique_path("archivr-core-init"); + let archive_parent = root.join("archive"); + let store_path = root.join("store"); + + let paths = initialize_archive(&archive_parent, &store_path, "Personal", false).unwrap(); + + assert_eq!(paths.archive_path, archive_parent.join(".archivr")); + assert!( + paths + .archive_path + .join(database::DATABASE_FILE_NAME) + .is_file() + ); + assert!(paths.store_path.join("raw").is_dir()); + assert!(paths.store_path.join("raw_tweets").is_dir()); + assert!(paths.store_path.join("structured").is_dir()); + assert!(paths.store_path.join("temp").is_dir()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn list_root_entries_returns_entry_details_and_runs() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + database::initialize_schema(&conn).unwrap(); + let user_id = database::ensure_default_user(&conn).unwrap(); + let run = database::create_archive_run(&conn, user_id, 1).unwrap(); + let item = database::create_archive_run_item( + &conn, + run.id, + None, + 0, + "https://example.com/saved", + Some("https://example.com/saved"), + "web", + "page", + ) + .unwrap(); + let source_identity_id = database::upsert_source_identity( + &conn, + "web", + "page", + Some("saved-article"), + Some("https://example.com/saved"), + "https://example.com/saved", + ) + .unwrap(); + let entry = database::create_archived_entry( + &conn, + &database::NewEntry { + source_identity_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "web".to_string(), + entity_kind: "page".to_string(), + title: Some("Saved Article".to_string()), + visibility: "private".to_string(), + representation_kind: "html".to_string(), + source_metadata_json: r#"{"source":"test"}"#.to_string(), + display_metadata_json: Some(r#"{"reading_time":"4m"}"#.to_string()), + }, + ) + .unwrap(); + let blob_id = database::upsert_blob( + &conn, + &database::BlobRecord { + sha256: "abc123".to_string(), + byte_size: 123, + mime_type: Some("text/html".to_string()), + extension: Some("html".to_string()), + raw_relpath: "raw/a/b/abc123.html".to_string(), + }, + ) + .unwrap(); + database::add_entry_artifact( + &conn, + &database::NewArtifact { + entry_id: entry.id, + artifact_role: "primary_media".to_string(), + storage_area: "raw".to_string(), + relpath: "raw/a/b/abc123.html".to_string(), + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + ) + .unwrap(); + database::complete_archive_run_item(&conn, item.id, entry.id).unwrap(); + database::finish_archive_run(&conn, run.id).unwrap(); + + let entries = list_root_entries(&conn).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].title.as_deref(), Some("Saved Article")); + assert_eq!(entries[0].artifact_count, 1); + + let detail = get_entry_detail(&conn, &entries[0].entry_uid) + .unwrap() + .unwrap(); + assert_eq!(detail.artifacts.len(), 1); + assert_eq!(detail.artifacts[0].artifact_role, "primary_media"); + + let runs = list_runs(&conn).unwrap(); + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].status, "completed"); + } +} diff --git a/crates/archivr-core/src/database.rs b/crates/archivr-core/src/database.rs new file mode 100644 index 0000000..5ec3f74 --- /dev/null +++ b/crates/archivr-core/src/database.rs @@ -0,0 +1,1000 @@ +use anyhow::{Context, Result, bail}; +use chrono::Utc; +use rusqlite::{Connection, OptionalExtension, params}; +use std::path::{Path, PathBuf}; +use uuid::Uuid; + +pub const DATABASE_FILE_NAME: &str = "archivr.sqlite"; +pub const DEFAULT_USERNAME: &str = "local-admin"; + +#[derive(Debug, Clone)] +pub struct ArchiveRun { + pub id: i64, + pub run_uid: String, +} + +#[derive(Debug, Clone)] +pub struct ArchiveRunItem { + pub id: i64, + pub item_uid: String, +} + +#[derive(Debug, Clone)] +pub struct ArchivedEntry { + pub id: i64, + pub entry_uid: String, + pub structured_root_relpath: String, +} + +#[derive(Debug, Clone)] +pub struct BlobRecord { + pub sha256: String, + pub byte_size: i64, + pub mime_type: Option, + pub extension: Option, + pub raw_relpath: String, +} + +#[derive(Debug, Clone)] +pub struct NewEntry { + pub source_identity_id: i64, + pub archive_run_id: i64, + pub parent_entry_id: Option, + pub root_entry_id: Option, + pub created_by_user_id: i64, + pub owned_by_user_id: i64, + pub source_kind: String, + pub entity_kind: String, + pub title: Option, + pub visibility: String, + pub representation_kind: String, + pub source_metadata_json: String, + pub display_metadata_json: Option, +} + +#[derive(Debug, Clone)] +pub struct NewArtifact { + pub entry_id: i64, + pub artifact_role: String, + pub storage_area: String, + pub relpath: String, + pub blob_id: Option, + pub logical_path: Option, + pub metadata_json: Option, +} + +pub fn database_path(archive_path: &Path) -> PathBuf { + archive_path.join(DATABASE_FILE_NAME) +} + +pub fn open_or_initialize(archive_path: &Path) -> Result { + let conn = Connection::open(database_path(archive_path)).with_context(|| { + format!( + "failed to open archive database in {}", + archive_path.display() + ) + })?; + initialize_schema(&conn)?; + Ok(conn) +} + +pub fn initialize_schema(conn: &Connection) -> Result<()> { + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "foreign_keys", "ON")?; + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY, + user_uid TEXT NOT NULL UNIQUE, + username TEXT NOT NULL UNIQUE, + email TEXT UNIQUE, + password_hash TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('active', 'disabled')), + role TEXT NOT NULL CHECK (role IN ('admin', 'user')), + created_at TEXT NOT NULL, + last_login_at TEXT + ); + + CREATE TABLE IF NOT EXISTS instance_settings ( + id INTEGER PRIMARY KEY CHECK (id = 1), + public_index_enabled INTEGER NOT NULL DEFAULT 0 CHECK (public_index_enabled IN (0, 1)), + public_entry_content_enabled INTEGER NOT NULL DEFAULT 0 CHECK (public_entry_content_enabled IN (0, 1)), + public_archive_submission_enabled INTEGER NOT NULL DEFAULT 0 CHECK (public_archive_submission_enabled IN (0, 1)) + ); + + INSERT OR IGNORE INTO instance_settings ( + id, + public_index_enabled, + public_entry_content_enabled, + public_archive_submission_enabled + ) VALUES (1, 0, 0, 0); + + CREATE TABLE IF NOT EXISTS archive_runs ( + id INTEGER PRIMARY KEY, + run_uid TEXT NOT NULL UNIQUE, + created_by_user_id INTEGER NOT NULL REFERENCES users(id), + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT NOT NULL CHECK (status IN ('in_progress', 'completed', 'failed')), + requested_count INTEGER NOT NULL DEFAULT 0, + discovered_count INTEGER NOT NULL DEFAULT 0, + completed_count INTEGER NOT NULL DEFAULT 0, + failed_count INTEGER NOT NULL DEFAULT 0, + error_summary TEXT + ); + + CREATE TABLE IF NOT EXISTS archive_run_items ( + id INTEGER PRIMARY KEY, + run_id INTEGER NOT NULL REFERENCES archive_runs(id) ON DELETE CASCADE, + item_uid TEXT NOT NULL UNIQUE, + parent_item_id INTEGER REFERENCES archive_run_items(id), + ordinal INTEGER NOT NULL, + requested_locator TEXT NOT NULL, + canonical_locator TEXT, + source_kind TEXT NOT NULL, + entity_kind TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'in_progress', 'completed', 'failed')), + error_text TEXT, + produced_entry_id INTEGER REFERENCES archived_entries(id) + ); + + CREATE TABLE IF NOT EXISTS source_identities ( + id INTEGER PRIMARY KEY, + source_kind TEXT NOT NULL, + entity_kind TEXT NOT NULL, + external_id TEXT, + canonical_url TEXT, + normalized_locator TEXT NOT NULL, + identity_key TEXT NOT NULL UNIQUE + ); + + CREATE TABLE IF NOT EXISTS archived_entries ( + id INTEGER PRIMARY KEY, + entry_uid TEXT NOT NULL UNIQUE, + source_identity_id INTEGER NOT NULL REFERENCES source_identities(id), + archive_run_id INTEGER NOT NULL REFERENCES archive_runs(id), + parent_entry_id INTEGER REFERENCES archived_entries(id), + root_entry_id INTEGER REFERENCES archived_entries(id), + created_by_user_id INTEGER NOT NULL REFERENCES users(id), + owned_by_user_id INTEGER NOT NULL REFERENCES users(id), + source_kind TEXT NOT NULL, + entity_kind TEXT NOT NULL, + title TEXT, + visibility TEXT NOT NULL CHECK (visibility IN ('private', 'unlisted', 'public')), + archived_at TEXT NOT NULL, + original_published_at TEXT, + structured_root_relpath TEXT NOT NULL, + representation_kind TEXT NOT NULL, + source_metadata_json TEXT NOT NULL DEFAULT '{}', + display_metadata_json TEXT + ); + + CREATE TABLE IF NOT EXISTS blobs ( + id INTEGER PRIMARY KEY, + sha256 TEXT NOT NULL UNIQUE, + byte_size INTEGER NOT NULL, + mime_type TEXT, + extension TEXT, + raw_relpath TEXT NOT NULL, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS entry_artifacts ( + id INTEGER PRIMARY KEY, + entry_id INTEGER NOT NULL REFERENCES archived_entries(id) ON DELETE CASCADE, + artifact_role TEXT NOT NULL, + storage_area TEXT NOT NULL CHECK (storage_area IN ('raw', 'raw_tweets', 'structured')), + relpath TEXT NOT NULL, + blob_id INTEGER REFERENCES blobs(id), + logical_path TEXT, + metadata_json TEXT + ); + + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY, + tag_uid TEXT NOT NULL UNIQUE, + parent_tag_id INTEGER REFERENCES tags(id), + name TEXT NOT NULL, + slug TEXT NOT NULL, + full_path TEXT NOT NULL UNIQUE + ); + + CREATE TABLE IF NOT EXISTS entry_tag_assignments ( + entry_id INTEGER NOT NULL REFERENCES archived_entries(id) ON DELETE CASCADE, + tag_id INTEGER NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY (entry_id, tag_id) + ); + + CREATE INDEX IF NOT EXISTS idx_archive_run_items_run_id ON archive_run_items(run_id); + CREATE INDEX IF NOT EXISTS idx_archived_entries_source_identity_id ON archived_entries(source_identity_id); + CREATE INDEX IF NOT EXISTS idx_archived_entries_created_by_user_id ON archived_entries(created_by_user_id); + CREATE INDEX IF NOT EXISTS idx_archived_entries_parent_entry_id ON archived_entries(parent_entry_id); + CREATE INDEX IF NOT EXISTS idx_archived_entries_root_entry_id ON archived_entries(root_entry_id); + CREATE INDEX IF NOT EXISTS idx_archived_entries_visibility ON archived_entries(visibility); + CREATE INDEX IF NOT EXISTS idx_entry_artifacts_entry_id ON entry_artifacts(entry_id); + CREATE INDEX IF NOT EXISTS idx_entry_artifacts_blob_id ON entry_artifacts(blob_id); + CREATE INDEX IF NOT EXISTS idx_tags_parent_tag_id ON tags(parent_tag_id); + "#, + )?; + Ok(()) +} + +pub fn ensure_default_user(conn: &Connection) -> Result { + if let Some(id) = conn + .query_row( + "SELECT id FROM users WHERE username = ?1", + [DEFAULT_USERNAME], + |row| row.get(0), + ) + .optional()? + { + return Ok(id); + } + + conn.execute( + "INSERT INTO users ( + user_uid, username, email, password_hash, status, role, created_at, last_login_at + ) VALUES (?1, ?2, NULL, ?3, 'active', 'admin', ?4, NULL)", + params![ + public_id("usr"), + DEFAULT_USERNAME, + "disabled-local-password", + now_timestamp() + ], + )?; + + Ok(conn.last_insert_rowid()) +} + +pub fn create_archive_run( + conn: &Connection, + created_by_user_id: i64, + requested_count: i64, +) -> Result { + let run_uid = public_id("run"); + conn.execute( + "INSERT INTO archive_runs ( + run_uid, created_by_user_id, started_at, status, requested_count + ) VALUES (?1, ?2, ?3, 'in_progress', ?4)", + params![ + run_uid, + created_by_user_id, + now_timestamp(), + requested_count + ], + )?; + + Ok(ArchiveRun { + id: conn.last_insert_rowid(), + run_uid, + }) +} + +pub fn create_archive_run_item( + conn: &Connection, + run_id: i64, + parent_item_id: Option, + ordinal: i64, + requested_locator: &str, + canonical_locator: Option<&str>, + source_kind: &str, + entity_kind: &str, +) -> Result { + let item_uid = public_id("item"); + conn.execute( + "INSERT INTO archive_run_items ( + run_id, item_uid, parent_item_id, ordinal, requested_locator, canonical_locator, + source_kind, entity_kind, status + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'in_progress')", + params![ + run_id, + item_uid, + parent_item_id, + ordinal, + requested_locator, + canonical_locator, + source_kind, + entity_kind + ], + )?; + + Ok(ArchiveRunItem { + id: conn.last_insert_rowid(), + item_uid, + }) +} + +pub fn complete_archive_run_item( + conn: &Connection, + item_id: i64, + produced_entry_id: i64, +) -> Result<()> { + conn.execute( + "UPDATE archive_run_items + SET status = 'completed', produced_entry_id = ?1, error_text = NULL + WHERE id = ?2", + params![produced_entry_id, item_id], + )?; + refresh_run_counters(conn, run_id_for_item(conn, item_id)?)?; + Ok(()) +} + +pub fn fail_archive_run_item(conn: &Connection, item_id: i64, error_text: &str) -> Result<()> { + conn.execute( + "UPDATE archive_run_items + SET status = 'failed', error_text = ?1 + WHERE id = ?2", + params![error_text, item_id], + )?; + refresh_run_counters(conn, run_id_for_item(conn, item_id)?)?; + Ok(()) +} + +pub fn finish_archive_run(conn: &Connection, run_id: i64) -> Result<()> { + refresh_run_counters(conn, run_id)?; + let failed_count: i64 = conn.query_row( + "SELECT failed_count FROM archive_runs WHERE id = ?1", + [run_id], + |row| row.get(0), + )?; + let status = if failed_count > 0 { + "failed" + } else { + "completed" + }; + conn.execute( + "UPDATE archive_runs SET status = ?1, finished_at = ?2 WHERE id = ?3", + params![status, now_timestamp(), run_id], + )?; + Ok(()) +} + +pub fn fail_archive_run(conn: &Connection, run_id: i64, error_summary: &str) -> Result<()> { + refresh_run_counters(conn, run_id)?; + conn.execute( + "UPDATE archive_runs + SET status = 'failed', finished_at = ?1, error_summary = ?2 + WHERE id = ?3", + params![now_timestamp(), error_summary, run_id], + )?; + Ok(()) +} + +pub fn upsert_source_identity( + conn: &Connection, + source_kind: &str, + entity_kind: &str, + external_id: Option<&str>, + canonical_url: Option<&str>, + normalized_locator: &str, +) -> Result { + let identity_key = identity_key( + source_kind, + entity_kind, + external_id, + canonical_url, + normalized_locator, + ); + conn.execute( + "INSERT OR IGNORE INTO source_identities ( + source_kind, entity_kind, external_id, canonical_url, normalized_locator, identity_key + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + source_kind, + entity_kind, + external_id, + canonical_url, + normalized_locator, + identity_key + ], + )?; + + let id = conn.query_row( + "SELECT id FROM source_identities WHERE identity_key = ?1", + [identity_key], + |row| row.get(0), + )?; + Ok(id) +} + +pub fn upsert_blob(conn: &Connection, blob: &BlobRecord) -> Result { + conn.execute( + "INSERT OR IGNORE INTO blobs ( + sha256, byte_size, mime_type, extension, raw_relpath, created_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + blob.sha256, + blob.byte_size, + blob.mime_type, + blob.extension, + blob.raw_relpath, + now_timestamp() + ], + )?; + + let id = conn.query_row( + "SELECT id FROM blobs WHERE sha256 = ?1", + [blob.sha256.as_str()], + |row| row.get(0), + )?; + Ok(id) +} + +pub fn create_archived_entry(conn: &Connection, entry: &NewEntry) -> Result { + validate_visibility(&entry.visibility)?; + let entry_uid = public_id("entry"); + let structured_root_relpath = format!("structured/{entry_uid}"); + + conn.execute( + "INSERT INTO archived_entries ( + entry_uid, source_identity_id, archive_run_id, parent_entry_id, root_entry_id, + created_by_user_id, owned_by_user_id, source_kind, entity_kind, title, visibility, + archived_at, original_published_at, structured_root_relpath, representation_kind, + source_metadata_json, display_metadata_json + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, + ?12, NULL, ?13, ?14, ?15, ?16 + )", + params![ + entry_uid, + entry.source_identity_id, + entry.archive_run_id, + entry.parent_entry_id, + entry.root_entry_id, + entry.created_by_user_id, + entry.owned_by_user_id, + entry.source_kind, + entry.entity_kind, + entry.title, + entry.visibility, + now_timestamp(), + structured_root_relpath, + entry.representation_kind, + entry.source_metadata_json, + entry.display_metadata_json + ], + )?; + let id = conn.last_insert_rowid(); + + if entry.root_entry_id.is_none() { + conn.execute( + "UPDATE archived_entries SET root_entry_id = ?1 WHERE id = ?1", + [id], + )?; + } + + Ok(ArchivedEntry { + id, + entry_uid, + structured_root_relpath, + }) +} + +pub fn add_entry_artifact(conn: &Connection, artifact: &NewArtifact) -> Result { + conn.execute( + "INSERT INTO entry_artifacts ( + entry_id, artifact_role, storage_area, relpath, blob_id, logical_path, metadata_json + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + artifact.entry_id, + artifact.artifact_role, + artifact.storage_area, + artifact.relpath, + artifact.blob_id, + artifact.logical_path, + artifact.metadata_json + ], + )?; + Ok(conn.last_insert_rowid()) +} + +#[cfg(test)] +pub fn set_public_settings( + conn: &Connection, + public_index_enabled: bool, + public_entry_content_enabled: bool, + public_archive_submission_enabled: bool, +) -> Result<()> { + conn.execute( + "UPDATE instance_settings + SET public_index_enabled = ?1, + public_entry_content_enabled = ?2, + public_archive_submission_enabled = ?3 + WHERE id = 1", + params![ + public_index_enabled as i64, + public_entry_content_enabled as i64, + public_archive_submission_enabled as i64 + ], + )?; + Ok(()) +} + +#[cfg(test)] +pub fn public_index_entry_count(conn: &Connection) -> Result { + let count = conn.query_row( + "SELECT COUNT(*) + FROM archived_entries + WHERE parent_entry_id IS NULL + AND visibility = 'public' + AND (SELECT public_index_enabled FROM instance_settings WHERE id = 1) = 1 + AND (SELECT public_entry_content_enabled FROM instance_settings WHERE id = 1) = 1", + [], + |row| row.get(0), + )?; + Ok(count) +} + +#[cfg(test)] +pub fn main_archive_entry_count(conn: &Connection) -> Result { + let count = conn.query_row( + "SELECT COUNT(*) FROM archived_entries WHERE parent_entry_id IS NULL", + [], + |row| row.get(0), + )?; + Ok(count) +} + +#[cfg(test)] +pub fn create_tag_path(conn: &Connection, full_path: &str) -> Result { + let segments = normalized_tag_segments(full_path)?; + let mut parent_tag_id = None; + let mut current_path = String::new(); + let mut current_id = 0; + + for segment in segments { + current_path.push('/'); + current_path.push_str(segment); + + if let Some(id) = conn + .query_row( + "SELECT id FROM tags WHERE full_path = ?1", + [current_path.as_str()], + |row| row.get(0), + ) + .optional()? + { + current_id = id; + parent_tag_id = Some(id); + continue; + } + + conn.execute( + "INSERT INTO tags (tag_uid, parent_tag_id, name, slug, full_path) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + public_id("tag"), + parent_tag_id, + humanize_slug(segment), + segment, + current_path + ], + )?; + current_id = conn.last_insert_rowid(); + parent_tag_id = Some(current_id); + } + + Ok(current_id) +} + +#[cfg(test)] +pub fn assign_entry_to_tag(conn: &Connection, entry_id: i64, tag_id: i64) -> Result<()> { + conn.execute( + "INSERT OR IGNORE INTO entry_tag_assignments (entry_id, tag_id) + VALUES (?1, ?2)", + params![entry_id, tag_id], + )?; + Ok(()) +} + +#[cfg(test)] +pub fn entry_count_for_tag_path(conn: &Connection, full_path: &str) -> Result { + let count = conn.query_row( + "WITH RECURSIVE descendants(id) AS ( + SELECT id FROM tags WHERE full_path = ?1 + UNION ALL + SELECT child.id + FROM tags child + JOIN descendants parent ON child.parent_tag_id = parent.id + ) + SELECT COUNT(DISTINCT eta.entry_id) + FROM entry_tag_assignments eta + JOIN descendants d ON eta.tag_id = d.id", + [full_path], + |row| row.get(0), + )?; + Ok(count) +} + +fn refresh_run_counters(conn: &Connection, run_id: i64) -> Result<()> { + conn.execute( + "UPDATE archive_runs + SET discovered_count = (SELECT COUNT(*) FROM archive_run_items WHERE run_id = ?1), + completed_count = (SELECT COUNT(*) FROM archive_run_items WHERE run_id = ?1 AND status = 'completed'), + failed_count = (SELECT COUNT(*) FROM archive_run_items WHERE run_id = ?1 AND status = 'failed') + WHERE id = ?1", + [run_id], + )?; + Ok(()) +} + +fn run_id_for_item(conn: &Connection, item_id: i64) -> Result { + let run_id = conn.query_row( + "SELECT run_id FROM archive_run_items WHERE id = ?1", + [item_id], + |row| row.get(0), + )?; + Ok(run_id) +} + +fn public_id(prefix: &str) -> String { + format!("{prefix}_{}", Uuid::new_v4().simple()) +} + +fn now_timestamp() -> String { + Utc::now().to_rfc3339() +} + +fn identity_key( + source_kind: &str, + entity_kind: &str, + external_id: Option<&str>, + canonical_url: Option<&str>, + normalized_locator: &str, +) -> String { + let stable_locator = external_id.or(canonical_url).unwrap_or(normalized_locator); + format!("{source_kind}:{entity_kind}:{stable_locator}") +} + +fn validate_visibility(visibility: &str) -> Result<()> { + match visibility { + "private" | "unlisted" | "public" => Ok(()), + _ => bail!("invalid archived entry visibility: {visibility}"), + } +} + +#[cfg(test)] +fn normalized_tag_segments(full_path: &str) -> Result> { + let segments = full_path + .trim() + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect::>(); + + if segments.is_empty() { + bail!("tag path must contain at least one segment"); + } + + Ok(segments) +} + +#[cfg(test)] +fn humanize_slug(slug: &str) -> String { + slug.split('-') + .map(|part| { + let mut chars = part.chars(); + match chars.next() { + Some(first) => format!("{}{}", first.to_uppercase(), chars.as_str()), + None => String::new(), + } + }) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + env, fs, + time::{SystemTime, UNIX_EPOCH}, + }; + + fn conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + initialize_schema(&conn).unwrap(); + conn + } + + fn unique_db_path(prefix: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + env::temp_dir().join(format!("{prefix}-{nanos}-{}.sqlite", std::process::id())) + } + + fn create_entry_fixture( + conn: &Connection, + visibility: &str, + parent_entry_id: Option, + root_entry_id: Option, + ) -> ArchivedEntry { + let user_id = ensure_default_user(conn).unwrap(); + let run = create_archive_run(conn, user_id, 1).unwrap(); + let source_id = upsert_source_identity( + conn, + "youtube", + "video", + Some("video-1"), + Some("https://youtube.com/watch?v=video-1"), + "https://youtube.com/watch?v=video-1", + ) + .unwrap(); + + create_archived_entry( + conn, + &NewEntry { + source_identity_id: source_id, + archive_run_id: run.id, + parent_entry_id, + root_entry_id, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "youtube".to_string(), + entity_kind: "video".to_string(), + title: None, + visibility: visibility.to_string(), + representation_kind: "video".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ) + .unwrap() + } + + #[test] + fn schema_defaults_public_settings_to_private() { + let conn = conn(); + let defaults: (i64, i64, i64) = conn + .query_row( + "SELECT public_index_enabled, public_entry_content_enabled, public_archive_submission_enabled + FROM instance_settings WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + + assert_eq!(defaults, (0, 0, 0)); + } + + #[test] + fn file_database_uses_wal_journal_mode() { + let path = unique_db_path("archivr-wal-test"); + let conn = Connection::open(&path).unwrap(); + initialize_schema(&conn).unwrap(); + + let journal_mode: String = conn + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .unwrap(); + + assert_eq!(journal_mode, "wal"); + + drop(conn); + let _ = fs::remove_file(&path); + let _ = fs::remove_file(path.with_extension("sqlite-wal")); + let _ = fs::remove_file(path.with_extension("sqlite-shm")); + } + + #[test] + fn root_entry_sets_root_id_after_insert() { + let conn = conn(); + let entry = create_entry_fixture(&conn, "private", None, None); + let root_entry_id: i64 = conn + .query_row( + "SELECT root_entry_id FROM archived_entries WHERE id = ?1", + [entry.id], + |row| row.get(0), + ) + .unwrap(); + + assert_eq!(root_entry_id, entry.id); + } + + #[test] + fn rearchiving_reuses_source_identity_and_blob_but_creates_entries() { + let conn = conn(); + let user_id = ensure_default_user(&conn).unwrap(); + let blob = BlobRecord { + sha256: "abc123".to_string(), + byte_size: 123, + mime_type: Some("video/mp4".to_string()), + extension: Some("mp4".to_string()), + raw_relpath: "raw/a/b/abc123.mp4".to_string(), + }; + let blob_id = upsert_blob(&conn, &blob).unwrap(); + let duplicate_blob_id = upsert_blob(&conn, &blob).unwrap(); + assert_eq!(blob_id, duplicate_blob_id); + + let first_source_id = upsert_source_identity( + &conn, + "youtube", + "video", + Some("video-1"), + Some("https://youtube.com/watch?v=video-1"), + "https://youtube.com/watch?v=video-1", + ) + .unwrap(); + let second_source_id = upsert_source_identity( + &conn, + "youtube", + "video", + Some("video-1"), + Some("https://youtube.com/watch?v=video-1"), + "https://youtube.com/watch?v=video-1", + ) + .unwrap(); + assert_eq!(first_source_id, second_source_id); + + for _ in 0..2 { + let run = create_archive_run(&conn, user_id, 1).unwrap(); + let entry = create_archived_entry( + &conn, + &NewEntry { + source_identity_id: first_source_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "youtube".to_string(), + entity_kind: "video".to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: "video".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ) + .unwrap(); + add_entry_artifact( + &conn, + &NewArtifact { + entry_id: entry.id, + artifact_role: "primary_media".to_string(), + storage_area: "raw".to_string(), + relpath: blob.raw_relpath.clone(), + blob_id: Some(blob_id), + logical_path: None, + metadata_json: None, + }, + ) + .unwrap(); + } + + let entry_count: i64 = conn + .query_row("SELECT COUNT(*) FROM archived_entries", [], |row| { + row.get(0) + }) + .unwrap(); + let source_count: i64 = conn + .query_row("SELECT COUNT(*) FROM source_identities", [], |row| { + row.get(0) + }) + .unwrap(); + let blob_count: i64 = conn + .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get(0)) + .unwrap(); + + assert_eq!(entry_count, 2); + assert_eq!(source_count, 1); + assert_eq!(blob_count, 1); + } + + #[test] + fn source_identity_key_prefers_external_id_over_shared_canonical_url() { + let conn = conn(); + let first_source_id = upsert_source_identity( + &conn, + "x", + "tweet", + Some("tweet-1"), + Some("https://x.com/some-profile"), + "https://x.com/some-profile/status/tweet-1", + ) + .unwrap(); + let second_source_id = upsert_source_identity( + &conn, + "x", + "tweet", + Some("tweet-2"), + Some("https://x.com/some-profile"), + "https://x.com/some-profile/status/tweet-2", + ) + .unwrap(); + + assert_ne!(first_source_id, second_source_id); + } + + #[test] + fn run_items_refresh_progress_counters() { + let conn = conn(); + let user_id = ensure_default_user(&conn).unwrap(); + let run = create_archive_run(&conn, user_id, 2).unwrap(); + let source_id = + upsert_source_identity(&conn, "local", "file", None, None, "file:///a").unwrap(); + let entry = create_archived_entry( + &conn, + &NewEntry { + source_identity_id: source_id, + archive_run_id: run.id, + parent_entry_id: None, + root_entry_id: None, + created_by_user_id: user_id, + owned_by_user_id: user_id, + source_kind: "local".to_string(), + entity_kind: "file".to_string(), + title: None, + visibility: "private".to_string(), + representation_kind: "file".to_string(), + source_metadata_json: "{}".to_string(), + display_metadata_json: None, + }, + ) + .unwrap(); + let first = + create_archive_run_item(&conn, run.id, None, 0, "file:///a", None, "local", "file") + .unwrap(); + let second = + create_archive_run_item(&conn, run.id, None, 1, "file:///b", None, "local", "file") + .unwrap(); + + complete_archive_run_item(&conn, first.id, entry.id).unwrap(); + fail_archive_run_item(&conn, second.id, "copy failed").unwrap(); + finish_archive_run(&conn, run.id).unwrap(); + + let counters: (i64, i64, i64, String) = conn + .query_row( + "SELECT discovered_count, completed_count, failed_count, status + FROM archive_runs WHERE id = ?1", + [run.id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + + assert_eq!(counters, (2, 1, 1, "failed".to_string())); + } + + #[test] + fn main_archive_query_only_counts_roots() { + let conn = conn(); + let parent = create_entry_fixture(&conn, "private", None, None); + let _child = create_entry_fixture(&conn, "private", Some(parent.id), Some(parent.id)); + + assert_eq!(main_archive_entry_count(&conn).unwrap(), 1); + } + + #[test] + fn public_entries_require_instance_flags_and_public_visibility() { + let conn = conn(); + let _public = create_entry_fixture(&conn, "public", None, None); + let _private = create_entry_fixture(&conn, "private", None, None); + + assert_eq!(public_index_entry_count(&conn).unwrap(), 0); + + set_public_settings(&conn, true, false, false).unwrap(); + assert_eq!(public_index_entry_count(&conn).unwrap(), 0); + + set_public_settings(&conn, true, true, false).unwrap(); + assert_eq!(public_index_entry_count(&conn).unwrap(), 1); + } + + #[test] + fn hierarchical_tag_assignments_are_discoverable_through_ancestors() { + let conn = conn(); + let entry = create_entry_fixture(&conn, "private", None, None); + let tag_id = create_tag_path(&conn, "/sciences/computer-science/compilers").unwrap(); + assign_entry_to_tag(&conn, entry.id, tag_id).unwrap(); + + assert_eq!( + entry_count_for_tag_path(&conn, "/sciences/computer-science/compilers").unwrap(), + 1 + ); + assert_eq!( + entry_count_for_tag_path(&conn, "/sciences/computer-science").unwrap(), + 1 + ); + assert_eq!(entry_count_for_tag_path(&conn, "/sciences").unwrap(), 1); + } +} diff --git a/src/downloader/local.rs b/crates/archivr-core/src/downloader/local.rs similarity index 100% rename from src/downloader/local.rs rename to crates/archivr-core/src/downloader/local.rs diff --git a/src/downloader/mod.rs b/crates/archivr-core/src/downloader/mod.rs similarity index 100% rename from src/downloader/mod.rs rename to crates/archivr-core/src/downloader/mod.rs diff --git a/src/downloader/store.rs b/crates/archivr-core/src/downloader/store.rs similarity index 100% rename from src/downloader/store.rs rename to crates/archivr-core/src/downloader/store.rs diff --git a/src/downloader/tweets.rs b/crates/archivr-core/src/downloader/tweets.rs similarity index 99% rename from src/downloader/tweets.rs rename to crates/archivr-core/src/downloader/tweets.rs index dc430d6..22d8bff 100644 --- a/src/downloader/tweets.rs +++ b/crates/archivr-core/src/downloader/tweets.rs @@ -10,9 +10,7 @@ use std::{ sync::OnceLock, }; -use crate::twitter::parse_tweet_id; - -use super::store; +use crate::{downloader::store, twitter::parse_tweet_id}; /// Extracts a tweet ID from an archivr path like `"tweet:123"` by taking the /// last colon-separated segment and validating it as a numeric ID. diff --git a/src/downloader/ytdlp.rs b/crates/archivr-core/src/downloader/ytdlp.rs similarity index 100% rename from src/downloader/ytdlp.rs rename to crates/archivr-core/src/downloader/ytdlp.rs diff --git a/src/hash.rs b/crates/archivr-core/src/hash.rs similarity index 100% rename from src/hash.rs rename to crates/archivr-core/src/hash.rs diff --git a/crates/archivr-core/src/lib.rs b/crates/archivr-core/src/lib.rs new file mode 100644 index 0000000..f1114c1 --- /dev/null +++ b/crates/archivr-core/src/lib.rs @@ -0,0 +1,5 @@ +pub mod archive; +pub mod database; +pub mod downloader; +pub mod hash; +pub mod twitter; diff --git a/src/twitter.rs b/crates/archivr-core/src/twitter.rs similarity index 100% rename from src/twitter.rs rename to crates/archivr-core/src/twitter.rs diff --git a/crates/archivr-server/Cargo.toml b/crates/archivr-server/Cargo.toml new file mode 100644 index 0000000..b90f678 --- /dev/null +++ b/crates/archivr-server/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "archivr-server" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +archivr-core = { path = "../archivr-core" } +axum.workspace = true +serde.workspace = true +tokio.workspace = true +toml.workspace = true +tower-http.workspace = true + +[dev-dependencies] +tempfile.workspace = true +tower.workspace = true diff --git a/crates/archivr-server/src/main.rs b/crates/archivr-server/src/main.rs new file mode 100644 index 0000000..13eb826 --- /dev/null +++ b/crates/archivr-server/src/main.rs @@ -0,0 +1,20 @@ +mod registry; +mod routes; + +use anyhow::Result; +use std::{net::SocketAddr, path::PathBuf}; + +#[tokio::main] +async fn main() -> Result<()> { + let config_path = std::env::args() + .nth(1) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("archivr-server.toml")); + let registry = registry::load_registry(&config_path)?; + let app = routes::app(registry); + let addr = SocketAddr::from(([127, 0, 0, 1], 8080)); + let listener = tokio::net::TcpListener::bind(addr).await?; + println!("archivr-server listening on http://{addr}"); + axum::serve(listener, app).await?; + Ok(()) +} diff --git a/crates/archivr-server/src/registry.rs b/crates/archivr-server/src/registry.rs new file mode 100644 index 0000000..c186d01 --- /dev/null +++ b/crates/archivr-server/src/registry.rs @@ -0,0 +1,111 @@ +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct MountedArchive { + pub id: String, + pub label: String, + pub archive_path: PathBuf, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct ServerRegistry { + pub archives: Vec, +} + +pub fn load_registry(path: &Path) -> Result { + let contents = fs::read_to_string(path) + .with_context(|| format!("failed to read server registry {}", path.display()))?; + let registry = toml::from_str::(&contents) + .with_context(|| format!("failed to parse server registry {}", path.display()))?; + validate_registry(®istry)?; + Ok(registry) +} + +#[cfg(test)] +pub fn save_registry(path: &Path, registry: &ServerRegistry) -> Result<()> { + validate_registry(registry)?; + let contents = toml::to_string_pretty(registry)?; + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, contents)?; + Ok(()) +} + +pub fn validate_registry(registry: &ServerRegistry) -> Result<()> { + let mut ids = std::collections::HashSet::new(); + for archive in ®istry.archives { + if archive.id.trim().is_empty() { + bail!("archive id must not be empty"); + } + if !ids.insert(archive.id.as_str()) { + bail!("duplicate archive id: {}", archive.id); + } + if !archive.archive_path.ends_with(".archivr") { + bail!( + "mounted archive path must point at a .archivr directory: {}", + archive.archive_path.display() + ); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_round_trips_archives_from_toml() { + let temp = tempfile::tempdir().unwrap(); + let archive_path = temp.path().join("personal").join(".archivr"); + fs::create_dir_all(&archive_path).unwrap(); + fs::write(archive_path.join("name"), "Personal").unwrap(); + fs::write( + archive_path.join("store_path"), + temp.path().join("store").display().to_string(), + ) + .unwrap(); + + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "personal".to_string(), + label: "Personal".to_string(), + archive_path: archive_path.clone(), + }], + }; + let path = temp.path().join("server.toml"); + save_registry(&path, ®istry).unwrap(); + + let loaded = load_registry(&path).unwrap(); + + assert_eq!(loaded, registry); + } + + #[test] + fn registry_rejects_duplicate_archive_ids() { + let registry = ServerRegistry { + archives: vec![ + MountedArchive { + id: "personal".to_string(), + label: "Personal".to_string(), + archive_path: PathBuf::from("/tmp/a/.archivr"), + }, + MountedArchive { + id: "personal".to_string(), + label: "Duplicate".to_string(), + archive_path: PathBuf::from("/tmp/b/.archivr"), + }, + ], + }; + + let err = validate_registry(®istry).unwrap_err().to_string(); + + assert!(err.contains("duplicate archive id")); + } +} diff --git a/crates/archivr-server/src/routes.rs b/crates/archivr-server/src/routes.rs new file mode 100644 index 0000000..dff563b --- /dev/null +++ b/crates/archivr-server/src/routes.rs @@ -0,0 +1,168 @@ +use std::{path::PathBuf, sync::Arc}; + +use archivr_core::{archive, database}; +use axum::{ + Json, Router, + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::get, +}; +use tower_http::services::{ServeDir, ServeFile}; + +use crate::registry::{MountedArchive, ServerRegistry}; + +#[derive(Clone)] +pub struct AppState { + registry: Arc, +} + +pub fn app(registry: ServerRegistry) -> Router { + let state = AppState { + registry: Arc::new(registry), + }; + let static_dir = static_dir(); + + Router::new() + .route("/health", get(|| async { "ok" })) + .route("/api/archives", get(list_archives)) + .route("/api/archives/:archive_id/entries", get(list_entries)) + .route( + "/api/archives/:archive_id/entries/:entry_uid", + get(entry_detail), + ) + .route("/api/archives/:archive_id/runs", get(list_runs)) + .nest_service("/assets", ServeDir::new(&static_dir)) + .fallback_service(ServeFile::new(static_dir.join("index.html"))) + .with_state(state) +} + +fn static_dir() -> PathBuf { + std::env::var_os("ARCHIVR_STATIC_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("static")) +} + +async fn list_archives(State(state): State) -> Json> { + Json(state.registry.archives.clone()) +} + +async fn list_entries( + State(state): State, + Path(archive_id): Path, +) -> Result>, ApiError> { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + Ok(Json(archive::list_root_entries(&conn)?)) +} + +async fn entry_detail( + State(state): State, + Path((archive_id, entry_uid)): Path<(String, String)>, +) -> Result, ApiError> { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + let detail = archive::get_entry_detail(&conn, &entry_uid)? + .ok_or(ApiError::not_found("entry not found"))?; + Ok(Json(detail)) +} + +async fn list_runs( + State(state): State, + Path(archive_id): Path, +) -> Result>, ApiError> { + let mounted = mounted_archive(&state, &archive_id)?; + let conn = database::open_or_initialize(&mounted.archive_path)?; + Ok(Json(archive::list_runs(&conn)?)) +} + +fn mounted_archive<'a>( + state: &'a AppState, + archive_id: &str, +) -> Result<&'a MountedArchive, ApiError> { + state + .registry + .archives + .iter() + .find(|archive| archive.id == archive_id) + .ok_or(ApiError::not_found("archive not found")) +} + +#[derive(Debug)] +pub struct ApiError { + status: StatusCode, + message: String, +} + +impl ApiError { + fn not_found(message: &str) -> Self { + Self { + status: StatusCode::NOT_FOUND, + message: message.to_string(), + } + } +} + +impl From for ApiError +where + E: Into, +{ + fn from(error: E) -> Self { + let error = error.into(); + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: format!("{error:#}"), + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + (self.status, self.message).into_response() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use tower::ServiceExt; + + #[tokio::test] + async fn archives_endpoint_lists_mounted_archives() { + let registry = ServerRegistry { + archives: vec![MountedArchive { + id: "personal".to_string(), + label: "Personal".to_string(), + archive_path: std::path::PathBuf::from("/tmp/personal/.archivr"), + }], + }; + let response = app(registry) + .oneshot( + Request::builder() + .uri("/api/archives") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn missing_archive_returns_404() { + let response = app(ServerRegistry::default()) + .oneshot( + Request::builder() + .uri("/api/archives/missing/entries") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } +} diff --git a/crates/archivr-server/static/app.js b/crates/archivr-server/static/app.js new file mode 100644 index 0000000..68b1616 --- /dev/null +++ b/crates/archivr-server/static/app.js @@ -0,0 +1,218 @@ +const state = { + archives: [], + archiveId: null, + entries: [], + filteredEntries: [], + selectedEntryUid: null, +}; + +const archiveSwitcher = document.querySelector("#archive-switcher"); +const entriesBody = document.querySelector("#entries-body"); +const runsBody = document.querySelector("#runs-body"); +const contextBody = document.querySelector("#context-body"); +const navButtons = document.querySelectorAll(".nav-link"); +const searchInput = document.querySelector("#search"); +const resultCount = document.querySelector("#result-count"); +const adminArchives = document.querySelector("#admin-archives"); + +function formatBytes(bytes) { + if (!bytes) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + let size = bytes; + let unit = 0; + while (size >= 1024 && unit < units.length - 1) { + size /= 1024; + unit += 1; + } + return `${size.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`; +} + +function valueText(value) { + return value ?? ""; +} + +async function getJson(url) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}`); + } + return response.json(); +} + +function appendCell(row, text, className) { + const cell = document.createElement("td"); + if (className) cell.className = className; + cell.textContent = text; + row.append(cell); + return cell; +} + +function renderArchives() { + archiveSwitcher.innerHTML = ""; + for (const archive of state.archives) { + const option = document.createElement("option"); + option.value = archive.id; + option.textContent = archive.label; + archiveSwitcher.append(option); + } + archiveSwitcher.value = state.archiveId ?? ""; + + adminArchives.innerHTML = ""; + for (const archive of state.archives) { + const item = document.createElement("div"); + item.className = "admin-archive"; + const label = document.createElement("strong"); + label.textContent = archive.label; + const path = document.createElement("div"); + path.className = "muted"; + path.textContent = archive.archive_path; + item.append(label, path); + adminArchives.append(item); + } +} + +function applyEntryFilter() { + const query = searchInput.value.trim().toLowerCase(); + if (!query) { + state.filteredEntries = state.entries; + return; + } + state.filteredEntries = state.entries.filter((entry) => { + const haystack = [ + entry.title, + entry.original_url, + entry.entry_uid, + entry.source_kind, + entry.entity_kind, + entry.visibility, + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return haystack.includes(query); + }); +} + +function renderEntries() { + entriesBody.innerHTML = ""; + resultCount.textContent = `${state.filteredEntries.length} entries`; + + for (const entry of state.filteredEntries) { + const row = document.createElement("tr"); + row.tabIndex = 0; + row.dataset.entryUid = entry.entry_uid; + if (entry.entry_uid === state.selectedEntryUid) { + row.classList.add("is-selected"); + } + + appendCell(row, valueText(entry.archived_at)); + + const titleCell = appendCell(row, ""); + const title = document.createElement("span"); + title.className = "entry-title"; + title.textContent = valueText(entry.title) || valueText(entry.entry_uid); + titleCell.append(title); + + const typeCell = appendCell(row, ""); + const type = document.createElement("span"); + type.className = "type-pill"; + type.textContent = valueText(entry.entity_kind); + typeCell.append(type); + + appendCell(row, formatBytes(entry.total_artifact_bytes)); + appendCell(row, valueText(entry.original_url), "url-cell"); + + row.addEventListener("click", () => selectEntry(entry)); + row.addEventListener("keydown", (event) => { + if (event.key === "Enter") selectEntry(entry); + }); + entriesBody.append(row); + } +} + +function renderContextDetail(detail) { + contextBody.innerHTML = ""; + const title = document.createElement("strong"); + title.textContent = valueText(detail.summary.title) || valueText(detail.summary.entry_uid); + contextBody.append(title); + + const items = [ + ["Type", detail.summary.entity_kind], + ["Visibility", detail.summary.visibility], + ["Artifacts", detail.artifacts.length], + ["Structured root", detail.structured_root_relpath], + ]; + + for (const [label, value] of items) { + const item = document.createElement("div"); + item.className = "rail-item"; + item.textContent = `${label}: ${valueText(value)}`; + contextBody.append(item); + } +} + +async function selectEntry(entry) { + state.selectedEntryUid = entry.entry_uid; + renderEntries(); + const detail = await getJson(`/api/archives/${state.archiveId}/entries/${entry.entry_uid}`); + renderContextDetail(detail); +} + +async function loadRuns() { + const runs = await getJson(`/api/archives/${state.archiveId}/runs`); + runsBody.innerHTML = ""; + for (const run of runs) { + const row = document.createElement("tr"); + appendCell(row, valueText(run.started_at)); + appendCell(row, valueText(run.status)); + appendCell(row, String(run.requested_count)); + appendCell(row, String(run.completed_count)); + appendCell(row, String(run.failed_count)); + runsBody.append(row); + } +} + +async function loadEntries() { + state.entries = await getJson(`/api/archives/${state.archiveId}/entries`); + state.selectedEntryUid = null; + applyEntryFilter(); + renderEntries(); + contextBody.textContent = "Select an entry."; +} + +async function loadArchives() { + state.archives = await getJson("/api/archives"); + state.archiveId = state.archives[0]?.id ?? null; + renderArchives(); + if (state.archiveId) { + await loadEntries(); + await loadRuns(); + } else { + contextBody.textContent = "No archives mounted."; + resultCount.textContent = "0 entries"; + } +} + +archiveSwitcher.addEventListener("change", async () => { + state.archiveId = archiveSwitcher.value; + await loadEntries(); + await loadRuns(); +}); + +searchInput.addEventListener("input", () => { + applyEntryFilter(); + renderEntries(); +}); + +navButtons.forEach((button) => { + button.addEventListener("click", () => { + navButtons.forEach((candidate) => candidate.classList.remove("is-active")); + document.querySelectorAll(".view").forEach((view) => view.classList.remove("is-active")); + button.classList.add("is-active"); + document.querySelector(`#${button.dataset.view}-view`).classList.add("is-active"); + }); +}); + +loadArchives().catch((error) => { + contextBody.textContent = `Failed to load archives: ${error.message}`; +}); diff --git a/crates/archivr-server/static/index.html b/crates/archivr-server/static/index.html new file mode 100644 index 0000000..021ef15 --- /dev/null +++ b/crates/archivr-server/static/index.html @@ -0,0 +1,79 @@ + + + + + + Archivr + + + +
+
Archivr
+ + + +
+ +
+
+
+ +
+
+ +
+ + + + + + + + + + + + + + + + + + +
AddedTitleTypeSizeOriginal URL
+
+ +
+ + + + + + + + + + + +
StartedStatusRequestedCompletedFailed
+
+ +
+

Mounted Archives

+
+
+
+ + +
+ + + + diff --git a/crates/archivr-server/static/styles.css b/crates/archivr-server/static/styles.css new file mode 100644 index 0000000..c0e9216 --- /dev/null +++ b/crates/archivr-server/static/styles.css @@ -0,0 +1,314 @@ +:root { + color-scheme: light; + --ink: #20251f; + --muted: #666a61; + --paper: #f5f0e7; + --paper-2: #e9e1d2; + --paper-3: #fffaf0; + --line: #d2c6b5; + --line-soft: #e5dccd; + --accent: #8d3f30; + --accent-2: #b78342; + --link: #245f72; + --top: #141d18; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-height: 100vh; + background: var(--paper); + color: var(--ink); + font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +button, +input, +select { + font: inherit; +} + +.topbar { + height: 56px; + display: grid; + grid-template-columns: auto minmax(180px, 250px) 1fr auto; + align-items: center; + gap: 18px; + padding: 0 18px; + background: var(--top); + color: #f7eedf; + border-bottom: 3px solid var(--accent); +} + +.brand { + font-family: Georgia, "Times New Roman", serif; + font-size: 28px; + line-height: 1; +} + +.archive-switcher { + width: 100%; + min-width: 0; + border: 1px solid rgba(247, 238, 223, 0.35); + background: #202b25; + color: #f7eedf; + padding: 7px 9px; +} + +.nav { + display: flex; + gap: 16px; + justify-content: flex-end; + min-width: 0; +} + +.nav-link { + border: 0; + background: transparent; + color: #d7cdbf; + cursor: pointer; + padding: 8px 0; +} + +.nav-link.is-active { + color: #fffaf0; + border-bottom: 1px solid #fffaf0; +} + +.capture-button { + border: 0; + background: #f7eedf; + color: var(--top); + padding: 9px 12px; + cursor: pointer; +} + +.app-shell { + height: calc(100vh - 56px); + display: grid; + grid-template-columns: minmax(0, 1fr) 320px; +} + +.workspace { + min-width: 0; + overflow: auto; +} + +.search-row { + min-height: 76px; + display: grid; + grid-template-columns: minmax(0, 1fr) max-content; + gap: 18px; + align-items: center; + padding: 14px 16px; + background: var(--paper-2); + border-bottom: 1px solid var(--line); +} + +.search-input { + width: 100%; + height: 46px; + border: 2px solid var(--ink); + background: var(--paper-3); + color: var(--ink); + padding: 0 14px; + font-size: 16px; + outline-offset: 3px; +} + +.result-count { + min-width: 76px; + color: var(--muted); + font-size: 13px; + text-align: right; +} + +.view { + display: none; +} + +.view.is-active { + display: block; +} + +.entry-table { + width: 100%; + border-collapse: collapse; + table-layout: fixed; + font-size: 13px; +} + +.entry-table th { + position: sticky; + top: 0; + z-index: 1; + text-align: left; + padding: 10px; + background: #ded5c7; + color: #5d625e; + border-bottom: 1px solid #cec4b5; + font-size: 11px; + text-transform: uppercase; +} + +.entry-table td { + padding: 10px; + border-bottom: 1px solid var(--line-soft); + vertical-align: top; +} + +.entry-table tr:nth-child(even) td { + background: #f2ede5; +} + +.entry-table tr:nth-child(odd) td { + background: var(--paper-3); +} + +.entry-table tr { + cursor: default; +} + +.entry-table tr.is-selected td { + background: #eee2d2; + border-top: 2px solid var(--accent); + border-bottom: 2px solid var(--accent); +} + +.col-added { + width: 178px; +} + +.col-title { + width: 38%; +} + +.col-type { + width: 130px; +} + +.col-size { + width: 110px; +} + +.col-url { + width: 34%; +} + +.entry-title { + color: var(--link); + font-weight: 750; +} + +.url-cell { + color: #555b55; + word-break: break-all; +} + +.type-pill { + display: inline-block; + padding: 2px 6px; + background: #d8e3df; + color: #275a5f; + border: 1px solid #bfd0ca; +} + +.context-rail { + border-left: 1px solid var(--line); + background: #efe7dc; + padding: 18px; + overflow: auto; +} + +.rail-title { + color: var(--accent); + font-weight: 800; + margin-bottom: 10px; + text-transform: uppercase; + font-size: 12px; + letter-spacing: 0; +} + +.rail-body { + color: var(--muted); + font-size: 14px; + line-height: 1.6; +} + +.rail-body strong { + color: var(--ink); +} + +.rail-item { + padding: 10px 0; + border-bottom: 1px solid var(--line); +} + +.admin-view { + padding: 22px; +} + +.admin-view h1 { + margin: 0 0 16px; + font-size: 18px; +} + +.admin-list { + display: grid; + gap: 10px; + max-width: 780px; +} + +.admin-archive { + border: 1px solid var(--line); + background: var(--paper-3); + padding: 12px; +} + +.muted { + color: var(--muted); +} + +@media (max-width: 900px) { + .topbar { + grid-template-columns: 1fr auto; + height: auto; + min-height: 56px; + padding: 12px; + } + + .archive-switcher, + .nav { + grid-column: 1 / -1; + } + + .nav { + justify-content: flex-start; + overflow-x: auto; + } + + .app-shell { + height: auto; + grid-template-columns: 1fr; + } + + .search-row { + grid-template-columns: 1fr; + } + + .result-count { + text-align: left; + } + + .context-rail { + border-left: 0; + border-top: 1px solid var(--line); + } + + .entry-table { + min-width: 860px; + } +} diff --git a/flake.nix b/flake.nix index a050caa..ca122fb 100644 --- a/flake.nix +++ b/flake.nix @@ -59,12 +59,45 @@ tweetPython = pkgs.python312.withPackages (ps: [ twitterApiClient ]); - archivr_unwrapped = pkgs.rustPlatform.buildRustPackage { - pname = "archivr"; - version = "0.1.0"; - src = pkgs.lib.cleanSource ./.; - cargoHash = "sha256-4m+4SMYA/rJ0eHEOc32zA2VdZI1pqzB5NenD0R0f2zM="; - nativeBuildInputs = [ pkgs.pkg-config ]; + version = "0.1.0"; + src = pkgs.lib.cleanSource ./.; + cargoLock = { + lockFile = ./Cargo.lock; + }; + nativeBuildInputs = [ pkgs.pkg-config ]; + archivr_cli_unwrapped = pkgs.rustPlatform.buildRustPackage { + pname = "archivr-cli"; + inherit + version + src + cargoLock + nativeBuildInputs + ; + cargoBuildFlags = [ + "-p" + "archivr-cli" + ]; + cargoTestFlags = [ + "-p" + "archivr-cli" + ]; + }; + archivr_server_unwrapped = pkgs.rustPlatform.buildRustPackage { + pname = "archivr-server"; + inherit + version + src + cargoLock + nativeBuildInputs + ; + cargoBuildFlags = [ + "-p" + "archivr-server" + ]; + cargoTestFlags = [ + "-p" + "archivr-server" + ]; }; archivr = pkgs.stdenv.mkDerivation { pname = "archivr-wrapped"; @@ -77,29 +110,43 @@ phases = [ "installPhase" ]; installPhase = '' mkdir -p $out/bin $out/libexec/archivr - cp -r ${archivr_unwrapped}/bin/* $out/bin/ + cp ${archivr_cli_unwrapped}/bin/archivr $out/libexec/archivr/archivr cp ${./vendor/twitter/scrape_user_tweet_contents.py} $out/libexec/archivr/scrape_user_tweet_contents.py chmod +x $out/libexec/archivr/scrape_user_tweet_contents.py - for f in $out/bin/*; do - mv "$f" "$f.orig" - makeWrapper "$f.orig" "$f" \ - --set ARCHIVR_YT_DLP ${pkgs.yt-dlp}/bin/yt-dlp \ - --set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \ - --set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr/scrape_user_tweet_contents.py \ - --prefix PATH : ${ - lib.makeBinPath [ - pkgs.yt-dlp - tweetPython - ] - } - done + makeWrapper $out/libexec/archivr/archivr $out/bin/archivr \ + --set ARCHIVR_YT_DLP ${pkgs.yt-dlp}/bin/yt-dlp \ + --set ARCHIVR_TWEET_PYTHON ${tweetPython}/bin/python3 \ + --set ARCHIVR_TWEET_SCRAPER $out/libexec/archivr/scrape_user_tweet_contents.py \ + --prefix PATH : ${ + lib.makeBinPath [ + pkgs.yt-dlp + tweetPython + ] + } + ''; + }; + archivr_server = pkgs.stdenv.mkDerivation { + pname = "archivr-server-wrapped"; + inherit version; + nativeBuildInputs = [ pkgs.makeWrapper ]; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin $out/libexec/archivr-server $out/share/archivr-server/static + cp ${archivr_server_unwrapped}/bin/archivr-server $out/libexec/archivr-server/archivr-server + cp -r ${./crates/archivr-server/static}/* $out/share/archivr-server/static/ + makeWrapper $out/libexec/archivr-server/archivr-server $out/bin/archivr-server \ + --set ARCHIVR_STATIC_DIR $out/share/archivr-server/static ''; }; in { default = archivr; archivr = archivr; - archivr-unwrapped = archivr_unwrapped; + archivr-cli = archivr; + archivr-cli-unwrapped = archivr_cli_unwrapped; + archivr-unwrapped = archivr_cli_unwrapped; + archivr-server = archivr_server; + archivr-server-unwrapped = archivr_server_unwrapped; } );