mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat: add db and multi-archive web UI foundation (#8)
* Add SQLite metadata database support * Implement archive metadata database * chore: let's guess cargoHash because there's something wrong with nixpkgs! * Gate test-only database helpers behind cfg(test) * Fix archive database row identity * Use serde for archive metadata JSON * Finalize archive runs at command level * Handle archive command errors without panics * Cover tweet entry metadata recording * Document static regex invariants * docs: add web UI design spec * docs: add web UI implementation plan * chore: move cli into workspace crate * chore: track workspace crates directory * refactor: extract archive core crate * refactor: add core archive opening APIs * refactor: rename taxonomy model to tags * feat: add archive query APIs * feat: add web server registry * feat: expose archive server APIs * feat: add archive table web UI * fix: complete web UI smoke path * docs: add architecture mental model * docs: remove private superpowers plans * nix: split cli and server packages * chore: remove PLAN.md
This commit is contained in:
parent
cc380ec5ba
commit
b56c969624
25 changed files with 3928 additions and 171 deletions
17
crates/archivr-cli/Cargo.toml
Normal file
17
crates/archivr-cli/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "archivr-cli"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "archivr"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
archivr-core = { path = "../archivr-core" }
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
clap.workspace = true
|
||||
regex.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde_json.workspace = true
|
||||
1321
crates/archivr-cli/src/main.rs
Normal file
1321
crates/archivr-cli/src/main.rs
Normal file
File diff suppressed because it is too large
Load diff
15
crates/archivr-core/Cargo.toml
Normal file
15
crates/archivr-core/Cargo.toml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[package]
|
||||
name = "archivr-core"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
hex.workspace = true
|
||||
regex.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha3.workspace = true
|
||||
uuid.workspace = true
|
||||
438
crates/archivr-core/src/archive.rs
Normal file
438
crates/archivr-core/src/archive.rs
Normal file
|
|
@ -0,0 +1,438 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use rusqlite::OptionalExtension;
|
||||
use std::{
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use crate::database;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ArchivePaths {
|
||||
pub archive_path: PathBuf,
|
||||
pub store_path: PathBuf,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct EntrySummary {
|
||||
pub entry_uid: String,
|
||||
pub archived_at: String,
|
||||
pub source_kind: String,
|
||||
pub entity_kind: String,
|
||||
pub title: Option<String>,
|
||||
pub visibility: String,
|
||||
pub original_url: Option<String>,
|
||||
pub artifact_count: i64,
|
||||
pub total_artifact_bytes: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct EntryDetail {
|
||||
pub summary: EntrySummary,
|
||||
pub structured_root_relpath: String,
|
||||
pub source_metadata_json: String,
|
||||
pub display_metadata_json: Option<String>,
|
||||
pub artifacts: Vec<EntryArtifactSummary>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct EntryArtifactSummary {
|
||||
pub artifact_role: String,
|
||||
pub storage_area: String,
|
||||
pub relpath: String,
|
||||
pub byte_size: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct RunSummary {
|
||||
pub run_uid: String,
|
||||
pub started_at: String,
|
||||
pub finished_at: Option<String>,
|
||||
pub status: String,
|
||||
pub requested_count: i64,
|
||||
pub discovered_count: i64,
|
||||
pub completed_count: i64,
|
||||
pub failed_count: i64,
|
||||
pub error_summary: Option<String>,
|
||||
}
|
||||
|
||||
pub fn find_archive_path_from(start: &Path) -> Result<Option<PathBuf>> {
|
||||
let mut dir = start.to_path_buf();
|
||||
loop {
|
||||
let candidate = dir.join(".archivr");
|
||||
if candidate.is_dir() {
|
||||
return Ok(Some(candidate));
|
||||
}
|
||||
if !dir.pop() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_archive_path() -> Result<Option<PathBuf>> {
|
||||
let cwd = env::current_dir().context("failed to read current working directory")?;
|
||||
find_archive_path_from(&cwd)
|
||||
}
|
||||
|
||||
pub fn read_archive_paths(archive_path: &Path) -> Result<ArchivePaths> {
|
||||
if !archive_path.is_dir() {
|
||||
bail!("archive path does not exist: {}", archive_path.display());
|
||||
}
|
||||
|
||||
let name = fs::read_to_string(archive_path.join("name"))
|
||||
.with_context(|| format!("failed to read archive name in {}", archive_path.display()))?
|
||||
.trim()
|
||||
.to_string();
|
||||
let store_path = fs::read_to_string(archive_path.join("store_path"))
|
||||
.with_context(|| format!("failed to read store path in {}", archive_path.display()))?;
|
||||
|
||||
Ok(ArchivePaths {
|
||||
archive_path: archive_path.to_path_buf(),
|
||||
store_path: PathBuf::from(store_path.trim()),
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn initialize_archive(
|
||||
archive_parent: &Path,
|
||||
store_path: &Path,
|
||||
archive_name: &str,
|
||||
force_with_info_removal: bool,
|
||||
) -> Result<ArchivePaths> {
|
||||
let archive_path = archive_parent.join(".archivr");
|
||||
|
||||
if archive_path.exists() {
|
||||
if !archive_path.is_dir() {
|
||||
bail!(
|
||||
"Archive path exists and is not a directory: {}",
|
||||
archive_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
if force_with_info_removal {
|
||||
fs::remove_dir_all(&archive_path)?;
|
||||
} else if fs::read_dir(&archive_path)?.next().is_some() {
|
||||
bail!(
|
||||
"Archive already exists at {} and is not empty. Use --force-with-info-removal to reinitialize.",
|
||||
archive_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if store_path.exists() && !force_with_info_removal {
|
||||
bail!("Store path already exists at {}", store_path.display());
|
||||
}
|
||||
|
||||
fs::create_dir_all(&archive_path)?;
|
||||
fs::create_dir_all(store_path)?;
|
||||
fs::write(archive_path.join("name"), archive_name)?;
|
||||
let canonical_store_path = store_path
|
||||
.canonicalize()
|
||||
.with_context(|| format!("failed to canonicalize {}", store_path.display()))?;
|
||||
fs::write(
|
||||
archive_path.join("store_path"),
|
||||
canonical_store_path
|
||||
.to_str()
|
||||
.context("store path is not valid UTF-8")?,
|
||||
)?;
|
||||
|
||||
initialize_store_directories(&canonical_store_path)?;
|
||||
let conn = database::open_or_initialize(&archive_path)?;
|
||||
let _ = database::ensure_default_user(&conn)?;
|
||||
|
||||
Ok(ArchivePaths {
|
||||
archive_path,
|
||||
store_path: canonical_store_path,
|
||||
name: archive_name.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn initialize_store_directories(store_path: &Path) -> Result<()> {
|
||||
fs::create_dir_all(store_path.join("raw"))?;
|
||||
fs::create_dir_all(store_path.join("raw_tweets"))?;
|
||||
fs::create_dir_all(store_path.join("structured"))?;
|
||||
fs::create_dir_all(store_path.join("temp"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_root_entries(conn: &rusqlite::Connection) -> Result<Vec<EntrySummary>> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT
|
||||
e.entry_uid,
|
||||
e.archived_at,
|
||||
e.source_kind,
|
||||
e.entity_kind,
|
||||
e.title,
|
||||
e.visibility,
|
||||
si.canonical_url,
|
||||
COUNT(ea.id) AS artifact_count,
|
||||
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes
|
||||
FROM archived_entries e
|
||||
JOIN source_identities si ON si.id = e.source_identity_id
|
||||
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
|
||||
LEFT JOIN blobs b ON b.id = ea.blob_id
|
||||
WHERE e.parent_entry_id IS NULL
|
||||
GROUP BY e.id
|
||||
ORDER BY e.archived_at DESC, e.id DESC",
|
||||
)?;
|
||||
|
||||
let entries = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(EntrySummary {
|
||||
entry_uid: row.get(0)?,
|
||||
archived_at: row.get(1)?,
|
||||
source_kind: row.get(2)?,
|
||||
entity_kind: row.get(3)?,
|
||||
title: row.get(4)?,
|
||||
visibility: row.get(5)?,
|
||||
original_url: row.get(6)?,
|
||||
artifact_count: row.get(7)?,
|
||||
total_artifact_bytes: row.get(8)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
pub fn get_entry_detail(
|
||||
conn: &rusqlite::Connection,
|
||||
entry_uid: &str,
|
||||
) -> Result<Option<EntryDetail>> {
|
||||
let Some((entry_id, structured_root_relpath, source_metadata_json, display_metadata_json)) =
|
||||
conn.query_row(
|
||||
"SELECT id, structured_root_relpath, source_metadata_json, display_metadata_json
|
||||
FROM archived_entries
|
||||
WHERE entry_uid = ?1",
|
||||
[entry_uid],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, i64>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, Option<String>>(3)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.optional()?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let summary = list_root_entries(conn)?
|
||||
.into_iter()
|
||||
.find(|entry| entry.entry_uid == entry_uid)
|
||||
.context("entry disappeared while loading detail")?;
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT ea.artifact_role, ea.storage_area, ea.relpath, b.byte_size
|
||||
FROM entry_artifacts ea
|
||||
LEFT JOIN blobs b ON b.id = ea.blob_id
|
||||
WHERE ea.entry_id = ?1
|
||||
ORDER BY ea.id ASC",
|
||||
)?;
|
||||
let artifacts = stmt
|
||||
.query_map([entry_id], |row| {
|
||||
Ok(EntryArtifactSummary {
|
||||
artifact_role: row.get(0)?,
|
||||
storage_area: row.get(1)?,
|
||||
relpath: row.get(2)?,
|
||||
byte_size: row.get(3)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
Ok(Some(EntryDetail {
|
||||
summary,
|
||||
structured_root_relpath,
|
||||
source_metadata_json,
|
||||
display_metadata_json,
|
||||
artifacts,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn list_runs(conn: &rusqlite::Connection) -> Result<Vec<RunSummary>> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT run_uid, started_at, finished_at, status, requested_count,
|
||||
discovered_count, completed_count, failed_count, error_summary
|
||||
FROM archive_runs
|
||||
ORDER BY started_at DESC, id DESC",
|
||||
)?;
|
||||
let runs = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(RunSummary {
|
||||
run_uid: row.get(0)?,
|
||||
started_at: row.get(1)?,
|
||||
finished_at: row.get(2)?,
|
||||
status: row.get(3)?,
|
||||
requested_count: row.get(4)?,
|
||||
discovered_count: row.get(5)?,
|
||||
completed_count: row.get(6)?,
|
||||
failed_count: row.get(7)?,
|
||||
error_summary: row.get(8)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
Ok(runs)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn unique_path(prefix: &str) -> PathBuf {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
env::temp_dir().join(format!("{prefix}-{nanos}-{}", std::process::id()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_archive_path_walks_up_to_dot_archivr() {
|
||||
let root = unique_path("archivr-core-find");
|
||||
let nested = root.join("a").join("b");
|
||||
fs::create_dir_all(root.join(".archivr")).unwrap();
|
||||
fs::create_dir_all(&nested).unwrap();
|
||||
|
||||
let found = find_archive_path_from(&nested).unwrap().unwrap();
|
||||
|
||||
assert_eq!(found, root.join(".archivr"));
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_archive_paths_returns_name_and_store_path() {
|
||||
let root = unique_path("archivr-core-open");
|
||||
let archive_path = root.join(".archivr");
|
||||
let store_path = root.join("store");
|
||||
fs::create_dir_all(&archive_path).unwrap();
|
||||
fs::create_dir_all(&store_path).unwrap();
|
||||
fs::write(archive_path.join("name"), "Personal").unwrap();
|
||||
fs::write(
|
||||
archive_path.join("store_path"),
|
||||
store_path.display().to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let paths = read_archive_paths(&archive_path).unwrap();
|
||||
|
||||
assert_eq!(paths.archive_path, archive_path);
|
||||
assert_eq!(paths.store_path, store_path);
|
||||
assert_eq!(paths.name, "Personal");
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initialize_archive_creates_database_store_and_metadata() {
|
||||
let root = unique_path("archivr-core-init");
|
||||
let archive_parent = root.join("archive");
|
||||
let store_path = root.join("store");
|
||||
|
||||
let paths = initialize_archive(&archive_parent, &store_path, "Personal", false).unwrap();
|
||||
|
||||
assert_eq!(paths.archive_path, archive_parent.join(".archivr"));
|
||||
assert!(
|
||||
paths
|
||||
.archive_path
|
||||
.join(database::DATABASE_FILE_NAME)
|
||||
.is_file()
|
||||
);
|
||||
assert!(paths.store_path.join("raw").is_dir());
|
||||
assert!(paths.store_path.join("raw_tweets").is_dir());
|
||||
assert!(paths.store_path.join("structured").is_dir());
|
||||
assert!(paths.store_path.join("temp").is_dir());
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_root_entries_returns_entry_details_and_runs() {
|
||||
let conn = rusqlite::Connection::open_in_memory().unwrap();
|
||||
database::initialize_schema(&conn).unwrap();
|
||||
let user_id = database::ensure_default_user(&conn).unwrap();
|
||||
let run = database::create_archive_run(&conn, user_id, 1).unwrap();
|
||||
let item = database::create_archive_run_item(
|
||||
&conn,
|
||||
run.id,
|
||||
None,
|
||||
0,
|
||||
"https://example.com/saved",
|
||||
Some("https://example.com/saved"),
|
||||
"web",
|
||||
"page",
|
||||
)
|
||||
.unwrap();
|
||||
let source_identity_id = database::upsert_source_identity(
|
||||
&conn,
|
||||
"web",
|
||||
"page",
|
||||
Some("saved-article"),
|
||||
Some("https://example.com/saved"),
|
||||
"https://example.com/saved",
|
||||
)
|
||||
.unwrap();
|
||||
let entry = database::create_archived_entry(
|
||||
&conn,
|
||||
&database::NewEntry {
|
||||
source_identity_id,
|
||||
archive_run_id: run.id,
|
||||
parent_entry_id: None,
|
||||
root_entry_id: None,
|
||||
created_by_user_id: user_id,
|
||||
owned_by_user_id: user_id,
|
||||
source_kind: "web".to_string(),
|
||||
entity_kind: "page".to_string(),
|
||||
title: Some("Saved Article".to_string()),
|
||||
visibility: "private".to_string(),
|
||||
representation_kind: "html".to_string(),
|
||||
source_metadata_json: r#"{"source":"test"}"#.to_string(),
|
||||
display_metadata_json: Some(r#"{"reading_time":"4m"}"#.to_string()),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let blob_id = database::upsert_blob(
|
||||
&conn,
|
||||
&database::BlobRecord {
|
||||
sha256: "abc123".to_string(),
|
||||
byte_size: 123,
|
||||
mime_type: Some("text/html".to_string()),
|
||||
extension: Some("html".to_string()),
|
||||
raw_relpath: "raw/a/b/abc123.html".to_string(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
database::add_entry_artifact(
|
||||
&conn,
|
||||
&database::NewArtifact {
|
||||
entry_id: entry.id,
|
||||
artifact_role: "primary_media".to_string(),
|
||||
storage_area: "raw".to_string(),
|
||||
relpath: "raw/a/b/abc123.html".to_string(),
|
||||
blob_id: Some(blob_id),
|
||||
logical_path: None,
|
||||
metadata_json: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
database::complete_archive_run_item(&conn, item.id, entry.id).unwrap();
|
||||
database::finish_archive_run(&conn, run.id).unwrap();
|
||||
|
||||
let entries = list_root_entries(&conn).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].title.as_deref(), Some("Saved Article"));
|
||||
assert_eq!(entries[0].artifact_count, 1);
|
||||
|
||||
let detail = get_entry_detail(&conn, &entries[0].entry_uid)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(detail.artifacts.len(), 1);
|
||||
assert_eq!(detail.artifacts[0].artifact_role, "primary_media");
|
||||
|
||||
let runs = list_runs(&conn).unwrap();
|
||||
assert_eq!(runs.len(), 1);
|
||||
assert_eq!(runs[0].status, "completed");
|
||||
}
|
||||
}
|
||||
1000
crates/archivr-core/src/database.rs
Normal file
1000
crates/archivr-core/src/database.rs
Normal file
File diff suppressed because it is too large
Load diff
28
crates/archivr-core/src/downloader/local.rs
Normal file
28
crates/archivr-core/src/downloader/local.rs
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use std::{path::Path, process::Command};
|
||||
|
||||
use crate::hash::hash_file;
|
||||
|
||||
pub fn save(path: String, store_path: &Path, timestamp: &String) -> Result<String> {
|
||||
println!("Saving path: {path}");
|
||||
|
||||
let temp_dir = store_path.join("temp").join(timestamp);
|
||||
std::fs::create_dir_all(&temp_dir)?;
|
||||
|
||||
let in_file = Path::new(path.trim_start_matches("file://"));
|
||||
let extension = in_file
|
||||
.extension()
|
||||
.map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy()));
|
||||
let out_file = temp_dir.join(format!("{timestamp}{extension}"));
|
||||
|
||||
let mut binding = Command::new("cp");
|
||||
let cmd = binding.arg(in_file).arg(&out_file);
|
||||
let out = cmd.output().with_context(|| "failed to spawn cp process")?;
|
||||
|
||||
if !out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
bail!("yt-dlp failed: {stderr}");
|
||||
}
|
||||
|
||||
hash_file(&out_file)
|
||||
}
|
||||
4
crates/archivr-core/src/downloader/mod.rs
Normal file
4
crates/archivr-core/src/downloader/mod.rs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
pub mod local;
|
||||
pub mod store;
|
||||
pub mod tweets;
|
||||
pub mod ytdlp;
|
||||
75
crates/archivr-core/src/downloader/store.rs
Normal file
75
crates/archivr-core/src/downloader/store.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
use anyhow::{Context, Result};
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use crate::hash::hash_file;
|
||||
|
||||
/// Moves `file` into the content-addressed raw store under `store_path`.
|
||||
///
|
||||
/// The destination path is derived from the file's SHA-256 hash:
|
||||
/// `raw/<first-char>/<second-char>/<hash><ext>`. If the destination already
|
||||
/// exists the source file is removed (deduplication); otherwise it is renamed.
|
||||
/// Returns the store-relative destination path.
|
||||
pub fn archive_staged_file(file: &Path, store_path: &Path) -> Result<PathBuf> {
|
||||
let hash = hash_file(file)?;
|
||||
let destination = raw_relative_path(file, &hash)?;
|
||||
let absolute_destination = store_path.join(&destination);
|
||||
|
||||
if let Some(parent) = absolute_destination.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
if absolute_destination.exists() {
|
||||
fs::remove_file(file)?;
|
||||
} else {
|
||||
fs::rename(file, &absolute_destination)?;
|
||||
}
|
||||
|
||||
Ok(destination)
|
||||
}
|
||||
|
||||
/// Computes the store-relative path for a file given its `hash`.
|
||||
/// The layout is `raw/<c1>/<c2>/<hash><ext>` where `c1`/`c2` are the first
|
||||
/// two characters of the hash, providing a two-level Trie.
|
||||
fn raw_relative_path(file: &Path, hash: &str) -> Result<PathBuf> {
|
||||
let mut chars = hash.chars();
|
||||
let first_letter = chars.next().context("hash must not be empty")?;
|
||||
let second_letter = chars
|
||||
.next()
|
||||
.context("hash must be at least two characters")?;
|
||||
let extension = file
|
||||
.extension()
|
||||
.map_or(String::new(), |ext| format!(".{}", ext.to_string_lossy()));
|
||||
|
||||
Ok(PathBuf::from("raw")
|
||||
.join(first_letter.to_string())
|
||||
.join(second_letter.to_string())
|
||||
.join(format!("{hash}{extension}")))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{env, fs};
|
||||
|
||||
#[test]
|
||||
fn test_archive_staged_file_moves_into_raw_store() {
|
||||
let root = env::temp_dir().join(format!("archivr-store-test-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
fs::create_dir_all(root.join("temp")).unwrap();
|
||||
|
||||
let staged = root.join("temp").join("photo.jpg");
|
||||
fs::write(&staged, b"image-bytes").unwrap();
|
||||
|
||||
let relative = archive_staged_file(&staged, &root).unwrap();
|
||||
let absolute = root.join(&relative);
|
||||
|
||||
assert!(absolute.is_file());
|
||||
assert!(!staged.exists());
|
||||
assert!(relative.starts_with("raw"));
|
||||
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
557
crates/archivr-core/src/downloader/tweets.rs
Normal file
557
crates/archivr-core/src/downloader/tweets.rs
Normal file
|
|
@ -0,0 +1,557 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use regex::Regex;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
env,
|
||||
ffi::OsString,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use crate::{downloader::store, twitter::parse_tweet_id};
|
||||
|
||||
/// Extracts a tweet ID from an archivr path like `"tweet:123"` by taking the
|
||||
/// last colon-separated segment and validating it as a numeric ID.
|
||||
fn tweet_id_from_path(path: &str) -> Option<String> {
|
||||
path.split(':').next_back().and_then(parse_tweet_id)
|
||||
}
|
||||
|
||||
/// Resolves `path` relative to `cwd` if it is not already absolute.
|
||||
fn absolutize_path_from_cwd(path: PathBuf, cwd: &Path) -> PathBuf {
|
||||
if path.is_absolute() {
|
||||
path
|
||||
} else {
|
||||
cwd.join(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the CLI argument list for the Python tweet scraper.
|
||||
/// When `thread` is true, recursive flags are added to follow reply chains.
|
||||
fn build_scraper_args(
|
||||
tweet_id: &str,
|
||||
thread: bool,
|
||||
output_dir: &Path,
|
||||
temp_dir: &Path,
|
||||
credentials_file: &Path,
|
||||
) -> Vec<String> {
|
||||
let mut args = vec![
|
||||
"--tweet-ids".to_string(),
|
||||
tweet_id.to_string(),
|
||||
"--output-dir".to_string(),
|
||||
output_dir.display().to_string(),
|
||||
"--media-dir".to_string(),
|
||||
temp_dir.join("media").display().to_string(),
|
||||
"--download-media".to_string(),
|
||||
"--credentials-file".to_string(),
|
||||
credentials_file.display().to_string(),
|
||||
];
|
||||
|
||||
if thread {
|
||||
args.push("--recursive-replied-to-tweets".to_string());
|
||||
args.push("--recursive-replied-to-tweets-quotes-retweets".to_string());
|
||||
args.push("--download-replied-to-tweets-media".to_string());
|
||||
} else {
|
||||
args.push("--no-recursive".to_string());
|
||||
}
|
||||
|
||||
args
|
||||
}
|
||||
|
||||
/// Archives a tweet (or full thread) identified by `path` (e.g. `"tweet:123"`).
|
||||
///
|
||||
/// Invokes the Python scraper, then moves all produced media assets into the
|
||||
/// content-addressed raw store and rewrites the JSON output to use the new
|
||||
/// store-relative paths. Returns `true` if new content was archived, `false`
|
||||
/// if the tweet was already present and `thread` is `false`.
|
||||
///
|
||||
/// Requires `ARCHIVR_TWITTER_CREDENTIALS_FILE` to be set. The scraper binary
|
||||
/// can be overridden via `ARCHIVR_TWEET_SCRAPER` and `ARCHIVR_TWEET_PYTHON`.
|
||||
pub fn archive(path: &str, thread: bool, store_path: &Path, timestamp: &str) -> Result<bool> {
|
||||
let invocation_cwd = env::current_dir().context("Failed to read current working directory")?;
|
||||
// Output directory for Tweet JSON files.
|
||||
let output_dir = store_path.join("raw_tweets");
|
||||
// Temporary directory for media assets downloaded by the scraper in `temp/...`.
|
||||
let temp_dir = store_path.join("temp").join(timestamp).join("tweets");
|
||||
let tweet_id = tweet_id_from_path(path).context("Invalid tweet ID")?;
|
||||
|
||||
fs::create_dir_all(&output_dir)?;
|
||||
fs::create_dir_all(&temp_dir)?;
|
||||
|
||||
// Path to the root - the to-be-archived tweet's JSON file.
|
||||
let root_json = output_dir.join(format!("tweet-{tweet_id}.json"));
|
||||
if !thread && root_json.exists() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let before = tweet_json_files(&output_dir)?;
|
||||
|
||||
let python = env::var_os("ARCHIVR_TWEET_PYTHON").unwrap_or_else(|| OsString::from("python3"));
|
||||
let scraper_path = env::var_os("ARCHIVR_TWEET_SCRAPER")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("vendor/twitter/scrape_user_tweet_contents.py"));
|
||||
let scraper_path = absolutize_path_from_cwd(scraper_path, &invocation_cwd);
|
||||
|
||||
let credentials_file = if let Some(credentials_file) =
|
||||
env::var_os("ARCHIVR_TWITTER_CREDENTIALS_FILE")
|
||||
{
|
||||
absolutize_path_from_cwd(PathBuf::from(credentials_file), &invocation_cwd)
|
||||
} else {
|
||||
bail!(
|
||||
"Twitter scraping requires ARCHIVR_TWITTER_CREDENTIALS_FILE to point to a cookies file."
|
||||
);
|
||||
};
|
||||
|
||||
if !credentials_file.is_file() {
|
||||
bail!(
|
||||
"Twitter credentials file not found: {}",
|
||||
credentials_file.display()
|
||||
);
|
||||
}
|
||||
|
||||
let mut cmd = Command::new(&python);
|
||||
cmd.current_dir(&temp_dir).arg(&scraper_path);
|
||||
for arg in build_scraper_args(&tweet_id, thread, &output_dir, &temp_dir, &credentials_file) {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
|
||||
let output = cmd.output().with_context(|| {
|
||||
format!(
|
||||
"Failed to spawn tweet scraper at {}",
|
||||
scraper_path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
bail!(
|
||||
"Tweet scraper failed.\nstdout:\n{}\nstderr:\n{}",
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
);
|
||||
}
|
||||
|
||||
if !root_json.exists() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
bail!(
|
||||
"Tweet scraper completed but did not create expected JSON file: {}\nstdout:\n{}\nstderr:\n{}",
|
||||
root_json.display(),
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
);
|
||||
}
|
||||
|
||||
cleanup_summary(&output_dir)?;
|
||||
let after = tweet_json_files(&output_dir)?;
|
||||
let new_jsons = new_tweet_jsons(&before, &after);
|
||||
rewrite_tweet_outputs(&new_jsons, &output_dir, &temp_dir, store_path)?;
|
||||
let _ = fs::remove_dir_all(store_path.join("temp").join(timestamp));
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Removes the `scraping_summary.json` file left by the scraper, if present.
|
||||
fn cleanup_summary(output_dir: &Path) -> Result<()> {
|
||||
let summary_path = output_dir.join("scraping_summary.json");
|
||||
if summary_path.exists() {
|
||||
fs::remove_file(summary_path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the set of `tweet-*.json` files present in `output_dir`.
|
||||
fn tweet_json_files(output_dir: &Path) -> Result<HashSet<PathBuf>> {
|
||||
let mut files = HashSet::new();
|
||||
|
||||
for entry in fs::read_dir(output_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.is_file()
|
||||
&& path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with("tweet-") && name.ends_with(".json"))
|
||||
{
|
||||
files.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Returns the sorted list of JSON files present in `after` but not in `before`.
|
||||
fn new_tweet_jsons(before: &HashSet<PathBuf>, after: &HashSet<PathBuf>) -> Vec<PathBuf> {
|
||||
let mut files = after.difference(before).cloned().collect::<Vec<_>>();
|
||||
files.sort();
|
||||
files
|
||||
}
|
||||
|
||||
/// Returns a lazily-compiled regex matching `"avatar_local_path": "..."` in JSON.
|
||||
fn avatar_regex() -> &'static Regex {
|
||||
static REGEX: OnceLock<Regex> = OnceLock::new();
|
||||
REGEX.get_or_init(|| Regex::new(r#""avatar_local_path": "([^"\n]+)""#).unwrap())
|
||||
}
|
||||
|
||||
/// Returns a lazily-compiled regex matching `"local_path": "..."` in JSON.
|
||||
fn media_regex() -> &'static Regex {
|
||||
static REGEX: OnceLock<Regex> = OnceLock::new();
|
||||
REGEX.get_or_init(|| Regex::new(r#"(?m)"local_path": "([^"\n]+)""#).unwrap())
|
||||
}
|
||||
|
||||
/// Rewrites asset paths in each newly-created JSON file, moving assets into
|
||||
/// the content-addressed store. Files are written back only if content changed.
|
||||
fn rewrite_tweet_outputs(
|
||||
tweet_jsons: &[PathBuf],
|
||||
output_dir: &Path,
|
||||
temp_dir: &Path,
|
||||
store_path: &Path,
|
||||
) -> Result<()> {
|
||||
let mut archived_assets = HashMap::new();
|
||||
|
||||
for path in tweet_jsons {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let rewritten = rewrite_json_asset_paths(
|
||||
&contents,
|
||||
output_dir,
|
||||
temp_dir,
|
||||
store_path,
|
||||
&mut archived_assets,
|
||||
)?;
|
||||
|
||||
if rewritten != contents {
|
||||
fs::write(path, rewritten)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rewrites all `avatar_local_path` and `local_path` references in `contents`,
|
||||
/// archiving each referenced file into the raw store and returning the updated
|
||||
/// JSON string. `archived_assets` is a cache to avoid re-archiving the same
|
||||
/// file when it is referenced by multiple tweets.
|
||||
fn rewrite_json_asset_paths(
|
||||
contents: &str,
|
||||
output_dir: &Path,
|
||||
temp_dir: &Path,
|
||||
store_path: &Path,
|
||||
archived_assets: &mut HashMap<String, String>,
|
||||
) -> Result<String> {
|
||||
let mut rewritten = contents.to_string();
|
||||
|
||||
for captures in avatar_regex().captures_iter(contents) {
|
||||
let old_path = captures[1].to_string();
|
||||
let new_path =
|
||||
archive_asset_reference(&old_path, output_dir, store_path, "avatar", archived_assets)?;
|
||||
rewritten = rewritten.replace(
|
||||
&format!(r#""avatar_local_path": "{old_path}""#),
|
||||
&format!(r#""avatar_local_path": "{new_path}""#),
|
||||
);
|
||||
}
|
||||
|
||||
for captures in media_regex().captures_iter(contents) {
|
||||
let old_path = captures[1].to_string();
|
||||
let new_path =
|
||||
archive_asset_reference(&old_path, temp_dir, store_path, "media", archived_assets)?;
|
||||
rewritten = rewritten.replace(
|
||||
&format!(r#""local_path": "{old_path}""#),
|
||||
&format!(r#""local_path": "{new_path}""#),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(rewritten)
|
||||
}
|
||||
|
||||
/// Archives the asset at `old_path` (relative to `base_dir`) into the raw store
|
||||
/// and returns its new store-relative path. Already-archived paths (starting
|
||||
/// with `"raw/"`) are returned unchanged. Results are cached in `archived_assets`
|
||||
/// by `"<kind>:<old_path>"` key to deduplicate work across TOML files.
|
||||
fn archive_asset_reference(
|
||||
old_path: &str,
|
||||
base_dir: &Path,
|
||||
store_path: &Path,
|
||||
kind: &str,
|
||||
archived_assets: &mut HashMap<String, String>,
|
||||
) -> Result<String> {
|
||||
if old_path.starts_with("raw/") {
|
||||
return Ok(old_path.to_string());
|
||||
}
|
||||
|
||||
let key = format!("{kind}:{old_path}");
|
||||
if let Some(existing) = archived_assets.get(&key) {
|
||||
return Ok(existing.clone());
|
||||
}
|
||||
|
||||
let absolute_path = base_dir.join(old_path);
|
||||
if !absolute_path.exists() {
|
||||
bail!(
|
||||
"Referenced tweet asset not found: {}",
|
||||
absolute_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
let relative_path = store::archive_staged_file(&absolute_path, store_path)?;
|
||||
let relative_path = relative_path.to_string_lossy().replace('\\', "/");
|
||||
archived_assets.insert(key, relative_path.clone());
|
||||
|
||||
Ok(relative_path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{
|
||||
sync::{Mutex, MutexGuard},
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
fn env_lock() -> MutexGuard<'static, ()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(())).lock().unwrap()
|
||||
}
|
||||
|
||||
fn unique_path(prefix: &str) -> PathBuf {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
env::temp_dir().join(format!("{prefix}-{nanos}-{}", std::process::id()))
|
||||
}
|
||||
|
||||
fn set_test_env(key: &str, value: impl AsRef<std::ffi::OsStr>) {
|
||||
unsafe {
|
||||
env::set_var(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_test_env(key: &str) {
|
||||
unsafe {
|
||||
env::remove_var(key);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_scraper_args_for_single_tweet() {
|
||||
let args = build_scraper_args(
|
||||
"1234567890",
|
||||
false,
|
||||
Path::new("/tmp/raw_tweets"),
|
||||
Path::new("/tmp/temp/tweets"),
|
||||
Path::new("/tmp/twitter-creds.txt"),
|
||||
);
|
||||
|
||||
assert!(args.contains(&"--tweet-ids".to_string()));
|
||||
assert!(args.contains(&"1234567890".to_string()));
|
||||
assert!(args.contains(&"--output-dir".to_string()));
|
||||
assert!(args.contains(&"--download-media".to_string()));
|
||||
assert!(args.contains(&"--credentials-file".to_string()));
|
||||
assert!(args.contains(&"--no-recursive".to_string()));
|
||||
assert!(!args.contains(&"--recursive-replied-to-tweets".to_string()));
|
||||
assert!(!args.contains(&"--recursive-replied-to-tweets-quotes-retweets".to_string()));
|
||||
assert!(!args.contains(&"--download-replied-to-tweets-media".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_scraper_args_for_thread() {
|
||||
let args = build_scraper_args(
|
||||
"1234567890",
|
||||
true,
|
||||
Path::new("/tmp/raw_tweets"),
|
||||
Path::new("/tmp/temp/tweets"),
|
||||
Path::new("/tmp/twitter-creds.txt"),
|
||||
);
|
||||
|
||||
assert!(args.contains(&"--recursive-replied-to-tweets".to_string()));
|
||||
assert!(args.contains(&"--recursive-replied-to-tweets-quotes-retweets".to_string()));
|
||||
assert!(args.contains(&"--download-replied-to-tweets-media".to_string()));
|
||||
assert!(!args.contains(&"--no-recursive".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_summary_removes_summary_only() {
|
||||
let output_dir = unique_path("archivr-tweet-summary");
|
||||
fs::create_dir_all(&output_dir).unwrap();
|
||||
fs::write(output_dir.join("scraping_summary.json"), "summary").unwrap();
|
||||
fs::write(output_dir.join("tweet-1.json"), "tweet").unwrap();
|
||||
|
||||
cleanup_summary(&output_dir).unwrap();
|
||||
|
||||
assert!(!output_dir.join("scraping_summary.json").exists());
|
||||
assert!(output_dir.join("tweet-1.json").exists());
|
||||
|
||||
let _ = fs::remove_dir_all(output_dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rewrite_json_asset_paths_rearchives_assets() {
|
||||
let store_path = unique_path("archivr-tweet-store");
|
||||
let output_dir = store_path.join("raw_tweets");
|
||||
let temp_dir = store_path.join("temp").join("ts").join("tweets");
|
||||
fs::create_dir_all(&output_dir).unwrap();
|
||||
fs::create_dir_all(temp_dir.join("media").join("avatars")).unwrap();
|
||||
fs::create_dir_all(temp_dir.join("media").join("123")).unwrap();
|
||||
|
||||
fs::write(
|
||||
temp_dir.join("media").join("avatars").join("avatar.jpg"),
|
||||
b"avatar",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
temp_dir.join("media").join("123").join("media_1.jpg"),
|
||||
b"media",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let contents = r#"{
|
||||
"entities": { "media": [{ "local_path": "media/123/media_1.jpg" }] },
|
||||
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/avatar.jpg" }
|
||||
}"#;
|
||||
|
||||
let rewritten = rewrite_json_asset_paths(
|
||||
contents,
|
||||
&output_dir,
|
||||
&temp_dir,
|
||||
&store_path,
|
||||
&mut HashMap::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(rewritten.contains(r#""avatar_local_path": "raw/"#));
|
||||
assert!(rewritten.contains(r#""local_path": "raw/"#));
|
||||
assert!(
|
||||
!temp_dir
|
||||
.join("media")
|
||||
.join("avatars")
|
||||
.join("avatar.jpg")
|
||||
.exists()
|
||||
);
|
||||
assert!(
|
||||
!temp_dir
|
||||
.join("media")
|
||||
.join("123")
|
||||
.join("media_1.jpg")
|
||||
.exists()
|
||||
);
|
||||
|
||||
let _ = fs::remove_dir_all(store_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_from_cwd_keeps_absolute_paths() {
|
||||
let path = absolutize_path_from_cwd(PathBuf::from("/tmp/creds.txt"), Path::new("/work"));
|
||||
assert_eq!(path, PathBuf::from("/tmp/creds.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_from_cwd_expands_relative_paths() {
|
||||
let path = absolutize_path_from_cwd(PathBuf::from("creds.txt"), Path::new("/work"));
|
||||
assert_eq!(path, PathBuf::from("/work/creds.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_skips_existing_flat_tweet() {
|
||||
let _guard = env_lock();
|
||||
let store_path = unique_path("archivr-tweet-skip");
|
||||
let output_dir = store_path.join("raw_tweets");
|
||||
fs::create_dir_all(&output_dir).unwrap();
|
||||
fs::create_dir_all(store_path.join("temp")).unwrap();
|
||||
fs::write(output_dir.join("tweet-123.json"), r#"{"id":"123"}"#).unwrap();
|
||||
|
||||
let credentials = store_path.join("creds.txt");
|
||||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||
|
||||
let archived = archive("tweet:123", false, &store_path, "ts").unwrap();
|
||||
|
||||
assert!(!archived);
|
||||
|
||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||
let _ = fs::remove_dir_all(store_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_flattens_tweets_and_rewrites_assets_with_stub_scraper() {
|
||||
let _guard = env_lock();
|
||||
let store_path = unique_path("archivr-tweet-integration");
|
||||
let output_dir = store_path.join("raw_tweets");
|
||||
fs::create_dir_all(&output_dir).unwrap();
|
||||
fs::create_dir_all(store_path.join("temp")).unwrap();
|
||||
|
||||
let credentials = store_path.join("creds.txt");
|
||||
fs::write(&credentials, "ct0=test;auth_token=test").unwrap();
|
||||
|
||||
let script = store_path.join("stub_scraper.sh");
|
||||
fs::write(
|
||||
&script,
|
||||
r#"#!/bin/sh
|
||||
set -eu
|
||||
|
||||
tweet_id=""
|
||||
output_dir=""
|
||||
media_dir=""
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--tweet-ids)
|
||||
tweet_id="$2"
|
||||
shift 2
|
||||
;;
|
||||
--output-dir)
|
||||
output_dir="$2"
|
||||
shift 2
|
||||
;;
|
||||
--media-dir)
|
||||
media_dir="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$output_dir" "$media_dir/avatars" "$media_dir/$tweet_id"
|
||||
printf 'avatar' > "$media_dir/avatars/author.jpg"
|
||||
printf 'media' > "$media_dir/$tweet_id/media_1.jpg"
|
||||
printf '{"summary":true}\n' > "$output_dir/scraping_summary.json"
|
||||
cat > "$output_dir/tweet-$tweet_id.json" <<EOF
|
||||
{
|
||||
"id": "$tweet_id",
|
||||
"entities": { "media": [{ "local_path": "media/$tweet_id/media_1.jpg" }] },
|
||||
"author": { "avatar_local_path": "../temp/ts/tweets/media/avatars/author.jpg" }
|
||||
}
|
||||
EOF
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
Command::new("chmod")
|
||||
.arg("+x")
|
||||
.arg(&script)
|
||||
.status()
|
||||
.unwrap();
|
||||
|
||||
set_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE", &credentials);
|
||||
set_test_env("ARCHIVR_TWEET_SCRAPER", &script);
|
||||
set_test_env("ARCHIVR_TWEET_PYTHON", "/bin/sh");
|
||||
|
||||
let archived = archive("tweet:123", false, &store_path, "ts").unwrap();
|
||||
let tweet_file = output_dir.join("tweet-123.json");
|
||||
let contents = fs::read_to_string(&tweet_file).unwrap();
|
||||
|
||||
assert!(archived);
|
||||
assert!(tweet_file.exists());
|
||||
assert!(!output_dir.join("scraping_summary.json").exists());
|
||||
assert!(contents.contains(r#""avatar_local_path": "raw/"#));
|
||||
assert!(contents.contains(r#""local_path": "raw/"#));
|
||||
assert!(!store_path.join("temp").join("ts").exists());
|
||||
|
||||
remove_test_env("ARCHIVR_TWITTER_CREDENTIALS_FILE");
|
||||
remove_test_env("ARCHIVR_TWEET_SCRAPER");
|
||||
remove_test_env("ARCHIVR_TWEET_PYTHON");
|
||||
let _ = fs::remove_dir_all(store_path);
|
||||
}
|
||||
}
|
||||
33
crates/archivr-core/src/downloader/ytdlp.rs
Normal file
33
crates/archivr-core/src/downloader/ytdlp.rs
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use std::{env, path::Path, process::Command};
|
||||
|
||||
use crate::hash::hash_file;
|
||||
|
||||
pub fn download(path: String, store_path: &Path, timestamp: &String) -> Result<String> {
|
||||
println!("Downloading with yt-dlp: {path}");
|
||||
|
||||
let ytdlp = env::var("ARCHIVR_YT_DLP").unwrap_or_else(|_| "yt-dlp".to_string());
|
||||
|
||||
let temp_dir = store_path.join("temp").join(timestamp);
|
||||
std::fs::create_dir_all(&temp_dir)?;
|
||||
|
||||
let out_file = temp_dir.join(format!("{timestamp}.mp4"));
|
||||
|
||||
let out = Command::new(&ytdlp)
|
||||
.arg(&path)
|
||||
.arg("-f")
|
||||
.arg("bestvideo+bestaudio/best")
|
||||
.arg("--merge-output-format")
|
||||
.arg("mp4")
|
||||
.arg("-o")
|
||||
.arg(&out_file)
|
||||
.output()
|
||||
.with_context(|| format!("failed to spawn {ytdlp} process"))?;
|
||||
|
||||
if !out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
bail!("yt-dlp failed: {stderr}");
|
||||
}
|
||||
|
||||
hash_file(&out_file)
|
||||
}
|
||||
17
crates/archivr-core/src/hash.rs
Normal file
17
crates/archivr-core/src/hash.rs
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
use anyhow::Result;
|
||||
use sha3::{Digest, Sha3_256};
|
||||
use std::{fs::File, io::Read, path::Path};
|
||||
|
||||
pub fn hash_file(path: &Path) -> Result<String> {
|
||||
let mut file = File::open(path)?;
|
||||
let mut buf = Vec::new();
|
||||
file.read_to_end(&mut buf)?;
|
||||
Ok(hash(String::from_utf8_lossy(&buf).to_string()))
|
||||
}
|
||||
|
||||
pub fn hash(path: String) -> String {
|
||||
let mut hasher = Sha3_256::new();
|
||||
hasher.update(path.as_bytes());
|
||||
let result = hasher.finalize();
|
||||
hex::encode(result)
|
||||
}
|
||||
5
crates/archivr-core/src/lib.rs
Normal file
5
crates/archivr-core/src/lib.rs
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
pub mod archive;
|
||||
pub mod database;
|
||||
pub mod downloader;
|
||||
pub mod hash;
|
||||
pub mod twitter;
|
||||
8
crates/archivr-core/src/twitter.rs
Normal file
8
crates/archivr-core/src/twitter.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/// Returns the tweet ID if `id` is non-empty and contains only ASCII digits.
|
||||
pub fn parse_tweet_id(id: &str) -> Option<String> {
|
||||
if !id.is_empty() && id.chars().all(|char| char.is_ascii_digit()) {
|
||||
Some(id.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
17
crates/archivr-server/Cargo.toml
Normal file
17
crates/archivr-server/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "archivr-server"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
archivr-core = { path = "../archivr-core" }
|
||||
axum.workspace = true
|
||||
serde.workspace = true
|
||||
tokio.workspace = true
|
||||
toml.workspace = true
|
||||
tower-http.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
tower.workspace = true
|
||||
20
crates/archivr-server/src/main.rs
Normal file
20
crates/archivr-server/src/main.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
mod registry;
|
||||
mod routes;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::{net::SocketAddr, path::PathBuf};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let config_path = std::env::args()
|
||||
.nth(1)
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("archivr-server.toml"));
|
||||
let registry = registry::load_registry(&config_path)?;
|
||||
let app = routes::app(registry);
|
||||
let addr = SocketAddr::from(([127, 0, 0, 1], 8080));
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
println!("archivr-server listening on http://{addr}");
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
111
crates/archivr-server/src/registry.rs
Normal file
111
crates/archivr-server/src/registry.rs
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct MountedArchive {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
pub archive_path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
|
||||
pub struct ServerRegistry {
|
||||
pub archives: Vec<MountedArchive>,
|
||||
}
|
||||
|
||||
pub fn load_registry(path: &Path) -> Result<ServerRegistry> {
|
||||
let contents = fs::read_to_string(path)
|
||||
.with_context(|| format!("failed to read server registry {}", path.display()))?;
|
||||
let registry = toml::from_str::<ServerRegistry>(&contents)
|
||||
.with_context(|| format!("failed to parse server registry {}", path.display()))?;
|
||||
validate_registry(®istry)?;
|
||||
Ok(registry)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn save_registry(path: &Path, registry: &ServerRegistry) -> Result<()> {
|
||||
validate_registry(registry)?;
|
||||
let contents = toml::to_string_pretty(registry)?;
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
fs::write(path, contents)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_registry(registry: &ServerRegistry) -> Result<()> {
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
for archive in ®istry.archives {
|
||||
if archive.id.trim().is_empty() {
|
||||
bail!("archive id must not be empty");
|
||||
}
|
||||
if !ids.insert(archive.id.as_str()) {
|
||||
bail!("duplicate archive id: {}", archive.id);
|
||||
}
|
||||
if !archive.archive_path.ends_with(".archivr") {
|
||||
bail!(
|
||||
"mounted archive path must point at a .archivr directory: {}",
|
||||
archive.archive_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn registry_round_trips_archives_from_toml() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let archive_path = temp.path().join("personal").join(".archivr");
|
||||
fs::create_dir_all(&archive_path).unwrap();
|
||||
fs::write(archive_path.join("name"), "Personal").unwrap();
|
||||
fs::write(
|
||||
archive_path.join("store_path"),
|
||||
temp.path().join("store").display().to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let registry = ServerRegistry {
|
||||
archives: vec![MountedArchive {
|
||||
id: "personal".to_string(),
|
||||
label: "Personal".to_string(),
|
||||
archive_path: archive_path.clone(),
|
||||
}],
|
||||
};
|
||||
let path = temp.path().join("server.toml");
|
||||
save_registry(&path, ®istry).unwrap();
|
||||
|
||||
let loaded = load_registry(&path).unwrap();
|
||||
|
||||
assert_eq!(loaded, registry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_rejects_duplicate_archive_ids() {
|
||||
let registry = ServerRegistry {
|
||||
archives: vec![
|
||||
MountedArchive {
|
||||
id: "personal".to_string(),
|
||||
label: "Personal".to_string(),
|
||||
archive_path: PathBuf::from("/tmp/a/.archivr"),
|
||||
},
|
||||
MountedArchive {
|
||||
id: "personal".to_string(),
|
||||
label: "Duplicate".to_string(),
|
||||
archive_path: PathBuf::from("/tmp/b/.archivr"),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let err = validate_registry(®istry).unwrap_err().to_string();
|
||||
|
||||
assert!(err.contains("duplicate archive id"));
|
||||
}
|
||||
}
|
||||
168
crates/archivr-server/src/routes.rs
Normal file
168
crates/archivr-server/src/routes.rs
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
use std::{path::PathBuf, sync::Arc};
|
||||
|
||||
use archivr_core::{archive, database};
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
routing::get,
|
||||
};
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
|
||||
use crate::registry::{MountedArchive, ServerRegistry};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
registry: Arc<ServerRegistry>,
|
||||
}
|
||||
|
||||
pub fn app(registry: ServerRegistry) -> Router {
|
||||
let state = AppState {
|
||||
registry: Arc::new(registry),
|
||||
};
|
||||
let static_dir = static_dir();
|
||||
|
||||
Router::new()
|
||||
.route("/health", get(|| async { "ok" }))
|
||||
.route("/api/archives", get(list_archives))
|
||||
.route("/api/archives/:archive_id/entries", get(list_entries))
|
||||
.route(
|
||||
"/api/archives/:archive_id/entries/:entry_uid",
|
||||
get(entry_detail),
|
||||
)
|
||||
.route("/api/archives/:archive_id/runs", get(list_runs))
|
||||
.nest_service("/assets", ServeDir::new(&static_dir))
|
||||
.fallback_service(ServeFile::new(static_dir.join("index.html")))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
fn static_dir() -> PathBuf {
|
||||
std::env::var_os("ARCHIVR_STATIC_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("static"))
|
||||
}
|
||||
|
||||
async fn list_archives(State(state): State<AppState>) -> Json<Vec<MountedArchive>> {
|
||||
Json(state.registry.archives.clone())
|
||||
}
|
||||
|
||||
async fn list_entries(
|
||||
State(state): State<AppState>,
|
||||
Path(archive_id): Path<String>,
|
||||
) -> Result<Json<Vec<archive::EntrySummary>>, ApiError> {
|
||||
let mounted = mounted_archive(&state, &archive_id)?;
|
||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||
Ok(Json(archive::list_root_entries(&conn)?))
|
||||
}
|
||||
|
||||
async fn entry_detail(
|
||||
State(state): State<AppState>,
|
||||
Path((archive_id, entry_uid)): Path<(String, String)>,
|
||||
) -> Result<Json<archive::EntryDetail>, ApiError> {
|
||||
let mounted = mounted_archive(&state, &archive_id)?;
|
||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||
let detail = archive::get_entry_detail(&conn, &entry_uid)?
|
||||
.ok_or(ApiError::not_found("entry not found"))?;
|
||||
Ok(Json(detail))
|
||||
}
|
||||
|
||||
async fn list_runs(
|
||||
State(state): State<AppState>,
|
||||
Path(archive_id): Path<String>,
|
||||
) -> Result<Json<Vec<archive::RunSummary>>, ApiError> {
|
||||
let mounted = mounted_archive(&state, &archive_id)?;
|
||||
let conn = database::open_or_initialize(&mounted.archive_path)?;
|
||||
Ok(Json(archive::list_runs(&conn)?))
|
||||
}
|
||||
|
||||
fn mounted_archive<'a>(
|
||||
state: &'a AppState,
|
||||
archive_id: &str,
|
||||
) -> Result<&'a MountedArchive, ApiError> {
|
||||
state
|
||||
.registry
|
||||
.archives
|
||||
.iter()
|
||||
.find(|archive| archive.id == archive_id)
|
||||
.ok_or(ApiError::not_found("archive not found"))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ApiError {
|
||||
status: StatusCode,
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl ApiError {
|
||||
fn not_found(message: &str) -> Self {
|
||||
Self {
|
||||
status: StatusCode::NOT_FOUND,
|
||||
message: message.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> From<E> for ApiError
|
||||
where
|
||||
E: Into<anyhow::Error>,
|
||||
{
|
||||
fn from(error: E) -> Self {
|
||||
let error = error.into();
|
||||
Self {
|
||||
status: StatusCode::INTERNAL_SERVER_ERROR,
|
||||
message: format!("{error:#}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
(self.status, self.message).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use tower::ServiceExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn archives_endpoint_lists_mounted_archives() {
|
||||
let registry = ServerRegistry {
|
||||
archives: vec![MountedArchive {
|
||||
id: "personal".to_string(),
|
||||
label: "Personal".to_string(),
|
||||
archive_path: std::path::PathBuf::from("/tmp/personal/.archivr"),
|
||||
}],
|
||||
};
|
||||
let response = app(registry)
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/archives")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_archive_returns_404() {
|
||||
let response = app(ServerRegistry::default())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/archives/missing/entries")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
}
|
||||
218
crates/archivr-server/static/app.js
Normal file
218
crates/archivr-server/static/app.js
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
const state = {
|
||||
archives: [],
|
||||
archiveId: null,
|
||||
entries: [],
|
||||
filteredEntries: [],
|
||||
selectedEntryUid: null,
|
||||
};
|
||||
|
||||
const archiveSwitcher = document.querySelector("#archive-switcher");
|
||||
const entriesBody = document.querySelector("#entries-body");
|
||||
const runsBody = document.querySelector("#runs-body");
|
||||
const contextBody = document.querySelector("#context-body");
|
||||
const navButtons = document.querySelectorAll(".nav-link");
|
||||
const searchInput = document.querySelector("#search");
|
||||
const resultCount = document.querySelector("#result-count");
|
||||
const adminArchives = document.querySelector("#admin-archives");
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!bytes) return "0 B";
|
||||
const units = ["B", "KB", "MB", "GB"];
|
||||
let size = bytes;
|
||||
let unit = 0;
|
||||
while (size >= 1024 && unit < units.length - 1) {
|
||||
size /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${size.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function valueText(value) {
|
||||
return value ?? "";
|
||||
}
|
||||
|
||||
async function getJson(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.status} ${response.statusText}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
function appendCell(row, text, className) {
|
||||
const cell = document.createElement("td");
|
||||
if (className) cell.className = className;
|
||||
cell.textContent = text;
|
||||
row.append(cell);
|
||||
return cell;
|
||||
}
|
||||
|
||||
function renderArchives() {
|
||||
archiveSwitcher.innerHTML = "";
|
||||
for (const archive of state.archives) {
|
||||
const option = document.createElement("option");
|
||||
option.value = archive.id;
|
||||
option.textContent = archive.label;
|
||||
archiveSwitcher.append(option);
|
||||
}
|
||||
archiveSwitcher.value = state.archiveId ?? "";
|
||||
|
||||
adminArchives.innerHTML = "";
|
||||
for (const archive of state.archives) {
|
||||
const item = document.createElement("div");
|
||||
item.className = "admin-archive";
|
||||
const label = document.createElement("strong");
|
||||
label.textContent = archive.label;
|
||||
const path = document.createElement("div");
|
||||
path.className = "muted";
|
||||
path.textContent = archive.archive_path;
|
||||
item.append(label, path);
|
||||
adminArchives.append(item);
|
||||
}
|
||||
}
|
||||
|
||||
function applyEntryFilter() {
|
||||
const query = searchInput.value.trim().toLowerCase();
|
||||
if (!query) {
|
||||
state.filteredEntries = state.entries;
|
||||
return;
|
||||
}
|
||||
state.filteredEntries = state.entries.filter((entry) => {
|
||||
const haystack = [
|
||||
entry.title,
|
||||
entry.original_url,
|
||||
entry.entry_uid,
|
||||
entry.source_kind,
|
||||
entry.entity_kind,
|
||||
entry.visibility,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
return haystack.includes(query);
|
||||
});
|
||||
}
|
||||
|
||||
function renderEntries() {
|
||||
entriesBody.innerHTML = "";
|
||||
resultCount.textContent = `${state.filteredEntries.length} entries`;
|
||||
|
||||
for (const entry of state.filteredEntries) {
|
||||
const row = document.createElement("tr");
|
||||
row.tabIndex = 0;
|
||||
row.dataset.entryUid = entry.entry_uid;
|
||||
if (entry.entry_uid === state.selectedEntryUid) {
|
||||
row.classList.add("is-selected");
|
||||
}
|
||||
|
||||
appendCell(row, valueText(entry.archived_at));
|
||||
|
||||
const titleCell = appendCell(row, "");
|
||||
const title = document.createElement("span");
|
||||
title.className = "entry-title";
|
||||
title.textContent = valueText(entry.title) || valueText(entry.entry_uid);
|
||||
titleCell.append(title);
|
||||
|
||||
const typeCell = appendCell(row, "");
|
||||
const type = document.createElement("span");
|
||||
type.className = "type-pill";
|
||||
type.textContent = valueText(entry.entity_kind);
|
||||
typeCell.append(type);
|
||||
|
||||
appendCell(row, formatBytes(entry.total_artifact_bytes));
|
||||
appendCell(row, valueText(entry.original_url), "url-cell");
|
||||
|
||||
row.addEventListener("click", () => selectEntry(entry));
|
||||
row.addEventListener("keydown", (event) => {
|
||||
if (event.key === "Enter") selectEntry(entry);
|
||||
});
|
||||
entriesBody.append(row);
|
||||
}
|
||||
}
|
||||
|
||||
function renderContextDetail(detail) {
|
||||
contextBody.innerHTML = "";
|
||||
const title = document.createElement("strong");
|
||||
title.textContent = valueText(detail.summary.title) || valueText(detail.summary.entry_uid);
|
||||
contextBody.append(title);
|
||||
|
||||
const items = [
|
||||
["Type", detail.summary.entity_kind],
|
||||
["Visibility", detail.summary.visibility],
|
||||
["Artifacts", detail.artifacts.length],
|
||||
["Structured root", detail.structured_root_relpath],
|
||||
];
|
||||
|
||||
for (const [label, value] of items) {
|
||||
const item = document.createElement("div");
|
||||
item.className = "rail-item";
|
||||
item.textContent = `${label}: ${valueText(value)}`;
|
||||
contextBody.append(item);
|
||||
}
|
||||
}
|
||||
|
||||
async function selectEntry(entry) {
|
||||
state.selectedEntryUid = entry.entry_uid;
|
||||
renderEntries();
|
||||
const detail = await getJson(`/api/archives/${state.archiveId}/entries/${entry.entry_uid}`);
|
||||
renderContextDetail(detail);
|
||||
}
|
||||
|
||||
async function loadRuns() {
|
||||
const runs = await getJson(`/api/archives/${state.archiveId}/runs`);
|
||||
runsBody.innerHTML = "";
|
||||
for (const run of runs) {
|
||||
const row = document.createElement("tr");
|
||||
appendCell(row, valueText(run.started_at));
|
||||
appendCell(row, valueText(run.status));
|
||||
appendCell(row, String(run.requested_count));
|
||||
appendCell(row, String(run.completed_count));
|
||||
appendCell(row, String(run.failed_count));
|
||||
runsBody.append(row);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadEntries() {
|
||||
state.entries = await getJson(`/api/archives/${state.archiveId}/entries`);
|
||||
state.selectedEntryUid = null;
|
||||
applyEntryFilter();
|
||||
renderEntries();
|
||||
contextBody.textContent = "Select an entry.";
|
||||
}
|
||||
|
||||
async function loadArchives() {
|
||||
state.archives = await getJson("/api/archives");
|
||||
state.archiveId = state.archives[0]?.id ?? null;
|
||||
renderArchives();
|
||||
if (state.archiveId) {
|
||||
await loadEntries();
|
||||
await loadRuns();
|
||||
} else {
|
||||
contextBody.textContent = "No archives mounted.";
|
||||
resultCount.textContent = "0 entries";
|
||||
}
|
||||
}
|
||||
|
||||
archiveSwitcher.addEventListener("change", async () => {
|
||||
state.archiveId = archiveSwitcher.value;
|
||||
await loadEntries();
|
||||
await loadRuns();
|
||||
});
|
||||
|
||||
searchInput.addEventListener("input", () => {
|
||||
applyEntryFilter();
|
||||
renderEntries();
|
||||
});
|
||||
|
||||
navButtons.forEach((button) => {
|
||||
button.addEventListener("click", () => {
|
||||
navButtons.forEach((candidate) => candidate.classList.remove("is-active"));
|
||||
document.querySelectorAll(".view").forEach((view) => view.classList.remove("is-active"));
|
||||
button.classList.add("is-active");
|
||||
document.querySelector(`#${button.dataset.view}-view`).classList.add("is-active");
|
||||
});
|
||||
});
|
||||
|
||||
loadArchives().catch((error) => {
|
||||
contextBody.textContent = `Failed to load archives: ${error.message}`;
|
||||
});
|
||||
79
crates/archivr-server/static/index.html
Normal file
79
crates/archivr-server/static/index.html
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Archivr</title>
|
||||
<link rel="stylesheet" href="/assets/styles.css">
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<div class="brand">Archivr</div>
|
||||
<select id="archive-switcher" class="archive-switcher" aria-label="Select archive"></select>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
<button class="nav-link is-active" data-view="archive">Archive</button>
|
||||
<button class="nav-link" data-view="runs">Runs</button>
|
||||
<button class="nav-link" data-view="admin">Admin</button>
|
||||
</nav>
|
||||
<button class="capture-button">+ Capture</button>
|
||||
</header>
|
||||
|
||||
<main class="app-shell">
|
||||
<section class="workspace">
|
||||
<div class="search-row">
|
||||
<input id="search" class="search-input" type="search" aria-label="Search archive" autocomplete="off">
|
||||
<div id="result-count" class="result-count"></div>
|
||||
</div>
|
||||
|
||||
<section id="archive-view" class="view is-active">
|
||||
<table class="entry-table">
|
||||
<colgroup>
|
||||
<col class="col-added">
|
||||
<col class="col-title">
|
||||
<col class="col-type">
|
||||
<col class="col-size">
|
||||
<col class="col-url">
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Added</th>
|
||||
<th>Title</th>
|
||||
<th>Type</th>
|
||||
<th>Size</th>
|
||||
<th>Original URL</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="entries-body"></tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section id="runs-view" class="view">
|
||||
<table class="entry-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Started</th>
|
||||
<th>Status</th>
|
||||
<th>Requested</th>
|
||||
<th>Completed</th>
|
||||
<th>Failed</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="runs-body"></tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section id="admin-view" class="view admin-view">
|
||||
<h1>Mounted Archives</h1>
|
||||
<div id="admin-archives" class="admin-list"></div>
|
||||
</section>
|
||||
</section>
|
||||
|
||||
<aside class="context-rail">
|
||||
<div class="rail-title">Context</div>
|
||||
<div id="context-body" class="rail-body">Select an entry.</div>
|
||||
</aside>
|
||||
</main>
|
||||
|
||||
<script type="module" src="/assets/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
314
crates/archivr-server/static/styles.css
Normal file
314
crates/archivr-server/static/styles.css
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
:root {
|
||||
color-scheme: light;
|
||||
--ink: #20251f;
|
||||
--muted: #666a61;
|
||||
--paper: #f5f0e7;
|
||||
--paper-2: #e9e1d2;
|
||||
--paper-3: #fffaf0;
|
||||
--line: #d2c6b5;
|
||||
--line-soft: #e5dccd;
|
||||
--accent: #8d3f30;
|
||||
--accent-2: #b78342;
|
||||
--link: #245f72;
|
||||
--top: #141d18;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
background: var(--paper);
|
||||
color: var(--ink);
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
|
||||
button,
|
||||
input,
|
||||
select {
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
.topbar {
|
||||
height: 56px;
|
||||
display: grid;
|
||||
grid-template-columns: auto minmax(180px, 250px) 1fr auto;
|
||||
align-items: center;
|
||||
gap: 18px;
|
||||
padding: 0 18px;
|
||||
background: var(--top);
|
||||
color: #f7eedf;
|
||||
border-bottom: 3px solid var(--accent);
|
||||
}
|
||||
|
||||
.brand {
|
||||
font-family: Georgia, "Times New Roman", serif;
|
||||
font-size: 28px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.archive-switcher {
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
border: 1px solid rgba(247, 238, 223, 0.35);
|
||||
background: #202b25;
|
||||
color: #f7eedf;
|
||||
padding: 7px 9px;
|
||||
}
|
||||
|
||||
.nav {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
justify-content: flex-end;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.nav-link {
|
||||
border: 0;
|
||||
background: transparent;
|
||||
color: #d7cdbf;
|
||||
cursor: pointer;
|
||||
padding: 8px 0;
|
||||
}
|
||||
|
||||
.nav-link.is-active {
|
||||
color: #fffaf0;
|
||||
border-bottom: 1px solid #fffaf0;
|
||||
}
|
||||
|
||||
.capture-button {
|
||||
border: 0;
|
||||
background: #f7eedf;
|
||||
color: var(--top);
|
||||
padding: 9px 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.app-shell {
|
||||
height: calc(100vh - 56px);
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) 320px;
|
||||
}
|
||||
|
||||
.workspace {
|
||||
min-width: 0;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.search-row {
|
||||
min-height: 76px;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) max-content;
|
||||
gap: 18px;
|
||||
align-items: center;
|
||||
padding: 14px 16px;
|
||||
background: var(--paper-2);
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.search-input {
|
||||
width: 100%;
|
||||
height: 46px;
|
||||
border: 2px solid var(--ink);
|
||||
background: var(--paper-3);
|
||||
color: var(--ink);
|
||||
padding: 0 14px;
|
||||
font-size: 16px;
|
||||
outline-offset: 3px;
|
||||
}
|
||||
|
||||
.result-count {
|
||||
min-width: 76px;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.view {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.view.is-active {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.entry-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
table-layout: fixed;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.entry-table th {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1;
|
||||
text-align: left;
|
||||
padding: 10px;
|
||||
background: #ded5c7;
|
||||
color: #5d625e;
|
||||
border-bottom: 1px solid #cec4b5;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.entry-table td {
|
||||
padding: 10px;
|
||||
border-bottom: 1px solid var(--line-soft);
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
.entry-table tr:nth-child(even) td {
|
||||
background: #f2ede5;
|
||||
}
|
||||
|
||||
.entry-table tr:nth-child(odd) td {
|
||||
background: var(--paper-3);
|
||||
}
|
||||
|
||||
.entry-table tr {
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.entry-table tr.is-selected td {
|
||||
background: #eee2d2;
|
||||
border-top: 2px solid var(--accent);
|
||||
border-bottom: 2px solid var(--accent);
|
||||
}
|
||||
|
||||
.col-added {
|
||||
width: 178px;
|
||||
}
|
||||
|
||||
.col-title {
|
||||
width: 38%;
|
||||
}
|
||||
|
||||
.col-type {
|
||||
width: 130px;
|
||||
}
|
||||
|
||||
.col-size {
|
||||
width: 110px;
|
||||
}
|
||||
|
||||
.col-url {
|
||||
width: 34%;
|
||||
}
|
||||
|
||||
.entry-title {
|
||||
color: var(--link);
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.url-cell {
|
||||
color: #555b55;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.type-pill {
|
||||
display: inline-block;
|
||||
padding: 2px 6px;
|
||||
background: #d8e3df;
|
||||
color: #275a5f;
|
||||
border: 1px solid #bfd0ca;
|
||||
}
|
||||
|
||||
.context-rail {
|
||||
border-left: 1px solid var(--line);
|
||||
background: #efe7dc;
|
||||
padding: 18px;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.rail-title {
|
||||
color: var(--accent);
|
||||
font-weight: 800;
|
||||
margin-bottom: 10px;
|
||||
text-transform: uppercase;
|
||||
font-size: 12px;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.rail-body {
|
||||
color: var(--muted);
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.rail-body strong {
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.rail-item {
|
||||
padding: 10px 0;
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.admin-view {
|
||||
padding: 22px;
|
||||
}
|
||||
|
||||
.admin-view h1 {
|
||||
margin: 0 0 16px;
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.admin-list {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
max-width: 780px;
|
||||
}
|
||||
|
||||
.admin-archive {
|
||||
border: 1px solid var(--line);
|
||||
background: var(--paper-3);
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.muted {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.topbar {
|
||||
grid-template-columns: 1fr auto;
|
||||
height: auto;
|
||||
min-height: 56px;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.archive-switcher,
|
||||
.nav {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.nav {
|
||||
justify-content: flex-start;
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.app-shell {
|
||||
height: auto;
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.search-row {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.result-count {
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.context-rail {
|
||||
border-left: 0;
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.entry-table {
|
||||
min-width: 860px;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue