mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
feat: add archive query APIs
This commit is contained in:
parent
4c13ce505f
commit
6390853c7b
1 changed files with 254 additions and 0 deletions
|
|
@ -1,4 +1,5 @@
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
|
use rusqlite::OptionalExtension;
|
||||||
use std::{
|
use std::{
|
||||||
env, fs,
|
env, fs,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
|
|
@ -13,6 +14,49 @@ pub struct ArchivePaths {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct EntrySummary {
|
||||||
|
pub entry_uid: String,
|
||||||
|
pub archived_at: String,
|
||||||
|
pub source_kind: String,
|
||||||
|
pub entity_kind: String,
|
||||||
|
pub title: Option<String>,
|
||||||
|
pub visibility: String,
|
||||||
|
pub original_url: Option<String>,
|
||||||
|
pub artifact_count: i64,
|
||||||
|
pub total_artifact_bytes: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct EntryDetail {
|
||||||
|
pub summary: EntrySummary,
|
||||||
|
pub structured_root_relpath: String,
|
||||||
|
pub source_metadata_json: String,
|
||||||
|
pub display_metadata_json: Option<String>,
|
||||||
|
pub artifacts: Vec<EntryArtifactSummary>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct EntryArtifactSummary {
|
||||||
|
pub artifact_role: String,
|
||||||
|
pub storage_area: String,
|
||||||
|
pub relpath: String,
|
||||||
|
pub byte_size: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct RunSummary {
|
||||||
|
pub run_uid: String,
|
||||||
|
pub started_at: String,
|
||||||
|
pub finished_at: Option<String>,
|
||||||
|
pub status: String,
|
||||||
|
pub requested_count: i64,
|
||||||
|
pub discovered_count: i64,
|
||||||
|
pub completed_count: i64,
|
||||||
|
pub failed_count: i64,
|
||||||
|
pub error_summary: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
pub fn find_archive_path_from(start: &Path) -> Result<Option<PathBuf>> {
|
pub fn find_archive_path_from(start: &Path) -> Result<Option<PathBuf>> {
|
||||||
let mut dir = start.to_path_buf();
|
let mut dir = start.to_path_buf();
|
||||||
loop {
|
loop {
|
||||||
|
|
@ -112,6 +156,128 @@ pub fn initialize_store_directories(store_path: &Path) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn list_root_entries(conn: &rusqlite::Connection) -> Result<Vec<EntrySummary>> {
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT
|
||||||
|
e.entry_uid,
|
||||||
|
e.archived_at,
|
||||||
|
e.source_kind,
|
||||||
|
e.entity_kind,
|
||||||
|
e.title,
|
||||||
|
e.visibility,
|
||||||
|
si.canonical_url,
|
||||||
|
COUNT(ea.id) AS artifact_count,
|
||||||
|
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes
|
||||||
|
FROM archived_entries e
|
||||||
|
JOIN source_identities si ON si.id = e.source_identity_id
|
||||||
|
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
|
||||||
|
LEFT JOIN blobs b ON b.id = ea.blob_id
|
||||||
|
WHERE e.parent_entry_id IS NULL
|
||||||
|
GROUP BY e.id
|
||||||
|
ORDER BY e.archived_at DESC, e.id DESC",
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let entries = stmt
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok(EntrySummary {
|
||||||
|
entry_uid: row.get(0)?,
|
||||||
|
archived_at: row.get(1)?,
|
||||||
|
source_kind: row.get(2)?,
|
||||||
|
entity_kind: row.get(3)?,
|
||||||
|
title: row.get(4)?,
|
||||||
|
visibility: row.get(5)?,
|
||||||
|
original_url: row.get(6)?,
|
||||||
|
artifact_count: row.get(7)?,
|
||||||
|
total_artifact_bytes: row.get(8)?,
|
||||||
|
})
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
Ok(entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_entry_detail(
|
||||||
|
conn: &rusqlite::Connection,
|
||||||
|
entry_uid: &str,
|
||||||
|
) -> Result<Option<EntryDetail>> {
|
||||||
|
let Some((entry_id, structured_root_relpath, source_metadata_json, display_metadata_json)) =
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, structured_root_relpath, source_metadata_json, display_metadata_json
|
||||||
|
FROM archived_entries
|
||||||
|
WHERE entry_uid = ?1",
|
||||||
|
[entry_uid],
|
||||||
|
|row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, i64>(0)?,
|
||||||
|
row.get::<_, String>(1)?,
|
||||||
|
row.get::<_, String>(2)?,
|
||||||
|
row.get::<_, Option<String>>(3)?,
|
||||||
|
))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.optional()?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
let summary = list_root_entries(conn)?
|
||||||
|
.into_iter()
|
||||||
|
.find(|entry| entry.entry_uid == entry_uid)
|
||||||
|
.context("entry disappeared while loading detail")?;
|
||||||
|
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT ea.artifact_role, ea.storage_area, ea.relpath, b.byte_size
|
||||||
|
FROM entry_artifacts ea
|
||||||
|
LEFT JOIN blobs b ON b.id = ea.blob_id
|
||||||
|
WHERE ea.entry_id = ?1
|
||||||
|
ORDER BY ea.id ASC",
|
||||||
|
)?;
|
||||||
|
let artifacts = stmt
|
||||||
|
.query_map([entry_id], |row| {
|
||||||
|
Ok(EntryArtifactSummary {
|
||||||
|
artifact_role: row.get(0)?,
|
||||||
|
storage_area: row.get(1)?,
|
||||||
|
relpath: row.get(2)?,
|
||||||
|
byte_size: row.get(3)?,
|
||||||
|
})
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
Ok(Some(EntryDetail {
|
||||||
|
summary,
|
||||||
|
structured_root_relpath,
|
||||||
|
source_metadata_json,
|
||||||
|
display_metadata_json,
|
||||||
|
artifacts,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_runs(conn: &rusqlite::Connection) -> Result<Vec<RunSummary>> {
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT run_uid, started_at, finished_at, status, requested_count,
|
||||||
|
discovered_count, completed_count, failed_count, error_summary
|
||||||
|
FROM archive_runs
|
||||||
|
ORDER BY started_at DESC, id DESC",
|
||||||
|
)?;
|
||||||
|
let runs = stmt
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok(RunSummary {
|
||||||
|
run_uid: row.get(0)?,
|
||||||
|
started_at: row.get(1)?,
|
||||||
|
finished_at: row.get(2)?,
|
||||||
|
status: row.get(3)?,
|
||||||
|
requested_count: row.get(4)?,
|
||||||
|
discovered_count: row.get(5)?,
|
||||||
|
completed_count: row.get(6)?,
|
||||||
|
failed_count: row.get(7)?,
|
||||||
|
error_summary: row.get(8)?,
|
||||||
|
})
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
Ok(runs)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -181,4 +347,92 @@ mod tests {
|
||||||
assert!(paths.store_path.join("temp").is_dir());
|
assert!(paths.store_path.join("temp").is_dir());
|
||||||
let _ = fs::remove_dir_all(root);
|
let _ = fs::remove_dir_all(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn list_root_entries_returns_entry_details_and_runs() {
|
||||||
|
let conn = rusqlite::Connection::open_in_memory().unwrap();
|
||||||
|
database::initialize_schema(&conn).unwrap();
|
||||||
|
let user_id = database::ensure_default_user(&conn).unwrap();
|
||||||
|
let run = database::create_archive_run(&conn, user_id, 1).unwrap();
|
||||||
|
let item = database::create_archive_run_item(
|
||||||
|
&conn,
|
||||||
|
run.id,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
"https://example.com/saved",
|
||||||
|
Some("https://example.com/saved"),
|
||||||
|
"web",
|
||||||
|
"page",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let source_identity_id = database::upsert_source_identity(
|
||||||
|
&conn,
|
||||||
|
"web",
|
||||||
|
"page",
|
||||||
|
Some("saved-article"),
|
||||||
|
Some("https://example.com/saved"),
|
||||||
|
"https://example.com/saved",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let entry = database::create_archived_entry(
|
||||||
|
&conn,
|
||||||
|
&database::NewEntry {
|
||||||
|
source_identity_id,
|
||||||
|
archive_run_id: run.id,
|
||||||
|
parent_entry_id: None,
|
||||||
|
root_entry_id: None,
|
||||||
|
created_by_user_id: user_id,
|
||||||
|
owned_by_user_id: user_id,
|
||||||
|
source_kind: "web".to_string(),
|
||||||
|
entity_kind: "page".to_string(),
|
||||||
|
title: Some("Saved Article".to_string()),
|
||||||
|
visibility: "private".to_string(),
|
||||||
|
representation_kind: "html".to_string(),
|
||||||
|
source_metadata_json: r#"{"source":"test"}"#.to_string(),
|
||||||
|
display_metadata_json: Some(r#"{"reading_time":"4m"}"#.to_string()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let blob_id = database::upsert_blob(
|
||||||
|
&conn,
|
||||||
|
&database::BlobRecord {
|
||||||
|
sha256: "abc123".to_string(),
|
||||||
|
byte_size: 123,
|
||||||
|
mime_type: Some("text/html".to_string()),
|
||||||
|
extension: Some("html".to_string()),
|
||||||
|
raw_relpath: "raw/a/b/abc123.html".to_string(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
database::add_entry_artifact(
|
||||||
|
&conn,
|
||||||
|
&database::NewArtifact {
|
||||||
|
entry_id: entry.id,
|
||||||
|
artifact_role: "primary_media".to_string(),
|
||||||
|
storage_area: "raw".to_string(),
|
||||||
|
relpath: "raw/a/b/abc123.html".to_string(),
|
||||||
|
blob_id: Some(blob_id),
|
||||||
|
logical_path: None,
|
||||||
|
metadata_json: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
database::complete_archive_run_item(&conn, item.id, entry.id).unwrap();
|
||||||
|
database::finish_archive_run(&conn, run.id).unwrap();
|
||||||
|
|
||||||
|
let entries = list_root_entries(&conn).unwrap();
|
||||||
|
assert_eq!(entries.len(), 1);
|
||||||
|
assert_eq!(entries[0].title.as_deref(), Some("Saved Article"));
|
||||||
|
assert_eq!(entries[0].artifact_count, 1);
|
||||||
|
|
||||||
|
let detail = get_entry_detail(&conn, &entries[0].entry_uid)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(detail.artifacts.len(), 1);
|
||||||
|
assert_eq!(detail.artifacts[0].artifact_role, "primary_media");
|
||||||
|
|
||||||
|
let runs = list_runs(&conn).unwrap();
|
||||||
|
assert_eq!(runs.len(), 1);
|
||||||
|
assert_eq!(runs[0].status, "completed");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue