thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00
Code Issues Activity

feat: precompute cached_bytes per entry in DB (#13)

Browse source 
Store how many bytes of each entry's artifacts are already on disk from
an earlier entry (content-addressed blob deduplication means shared
blobs are only stored once).

Design
------
- Add `cached_bytes INTEGER NOT NULL DEFAULT 0` to `archived_entries`
- Precompute at capture time via `database::refresh_entry_cached_bytes`
  called after all artifacts are saved for every capture path
  (web page, generic URL, tweet, yt-dlp/local)
- One-time migration in `initialize_schema`: detects missing column via
  PRAGMA table_info, ALTERs the table, then back-fills all existing rows
  with the correlated subquery
- `database::cascade_cached_bytes_after_delete` ready for when entry
  deletion is implemented; designed to run asynchronously after the
  delete is acknowledged to the user
- `cached_bytes` included in `EntrySummary` and all four SELECT paths
  (list_root_entries, search_entries, list_entries_for_collection,
  entries_for_tag) via the shared ENTRY_SELECT_COLS constant

Frontend
--------
- `EntryRow` shows a `% cached` sub-line under the size when non-zero,
  with a tooltip showing the raw cached byte count
- No separate API endpoint or extra fetch — value rides in the existing
  entries list response at zero extra query cost per read
This commit is contained in:
TheGeneralist 2026-07-03 11:56:55 +02:00 • committed by GitHub
parent c85cce579b
commit 90d17c9d25
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 161 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

15
crates/archivr-core/src/archive.rs
View file

@ -28,6 +28,8 @@ pub struct EntrySummary {
pub parent_entry_uid: Option<String>,
/// True if a `favicon` artifact exists for this entry.
pub has_favicon: bool,
/// Bytes of blobs already on disk from an earlier entry (precomputed at capture time).
pub cached_bytes: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
@ -206,7 +208,8 @@ pub fn list_root_entries(conn: &rusqlite::Connection, caller_bits: u32) -> Resul
COUNT(ea.id) AS artifact_count,
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
NULL AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon,
e.cached_bytes
FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -238,6 +241,7 @@ pub fn list_root_entries(conn: &rusqlite::Connection, caller_bits: u32) -> Resul
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -425,6 +429,7 @@ pub fn list_entries_for_collection(
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -539,7 +544,8 @@ const ENTRY_SELECT_COLS: &str =
e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, \
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \
parent.entry_uid AS parent_entry_uid, \
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon";
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, \
e.cached_bytes";
const ENTRY_FROM_JOINS: &str =
"FROM archived_entries e \
@ -649,6 +655,7 @@ pub fn search_entries(
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
@ -810,7 +817,8 @@ pub fn entries_for_tag(
e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count,
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
parent.entry_uid AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon,
e.cached_bytes
FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -835,6 +843,7 @@ pub fn entries_for_tag(
total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;

12
crates/archivr-core/src/capture.rs
View file

@ -751,7 +751,7 @@ pub fn perform_capture(
let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
}
record_media_entry(
let entry = record_media_entry(
&conn,
store_path,
user_id,
@ -765,6 +765,7 @@ pub fn perform_capture(
byte_size,
title_hint,
)?;
database::refresh_entry_cached_bytes(&conn, entry.id)?;
database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult {
run_uid: run.run_uid.clone(),
@ -904,6 +905,9 @@ pub fn perform_capture(
}
}
// 7. Store how many bytes this entry gets "for free" from earlier entries.
database::refresh_entry_cached_bytes(&conn, entry.id)?;
database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult {
run_uid: run.run_uid.clone(),
@ -942,7 +946,7 @@ pub fn perform_capture(
&timestamp,
) {
Ok(_) => {
record_tweet_entry(
let tweet_entry = record_tweet_entry(
&conn,
store_path,
user_id,
@ -952,6 +956,7 @@ pub fn perform_capture(
source,
&tweet_id,
)?;
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult {
run_uid: run.run_uid.clone(),
@ -1042,7 +1047,7 @@ pub fn perform_capture(
let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
}
record_media_entry(
let media_entry = record_media_entry(
&conn,
store_path,
user_id,
@ -1056,6 +1061,7 @@ pub fn perform_capture(
byte_size,
None, // title — populated in a later task
)?;
database::refresh_entry_cached_bytes(&conn, media_entry.id)?;
database::finish_archive_run(&conn, run.id)?;
Ok(CaptureResult {

111
crates/archivr-core/src/database.rs
View file

@ -251,7 +251,8 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> {
structured_root_relpath TEXT NOT NULL,
representation_kind TEXT NOT NULL,
source_metadata_json TEXT NOT NULL DEFAULT '{}',
display_metadata_json TEXT
display_metadata_json TEXT,
cached_bytes INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS blobs (
@ -350,6 +351,38 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> {
FROM archived_entries ae;
"#,
)?;
// Migration: add cached_bytes column to existing databases.
// New databases already have it from the DDL above; the column check is
// the idiomatic SQLite way to run a migration exactly once.
let column_exists: bool = conn.query_row(
"SELECT COUNT(*) FROM pragma_table_info('archived_entries') WHERE name = 'cached_bytes'",
[],
|row| row.get::<_, i64>(0),
)? > 0;
if !column_exists {
conn.execute_batch(
"ALTER TABLE archived_entries ADD COLUMN cached_bytes INTEGER NOT NULL DEFAULT 0;
UPDATE archived_entries
SET cached_bytes = (
SELECT COALESCE(SUM(b.byte_size), 0)
FROM entry_artifacts ea
JOIN blobs b ON b.id = ea.blob_id
WHERE ea.entry_id = archived_entries.id
AND ea.blob_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM entry_artifacts ea2
JOIN archived_entries e2 ON e2.id = ea2.entry_id
WHERE ea2.blob_id = ea.blob_id
AND (e2.archived_at < archived_entries.archived_at
OR (e2.archived_at = archived_entries.archived_at
AND e2.id < archived_entries.id))
)
);",
)?;
}
Ok(())
}
@ -1191,6 +1224,82 @@ pub fn upsert_source_identity(
Ok(id)
}
/// Computes and stores `cached_bytes` for a single entry.
///
/// Must be called after all artifacts for the entry have been inserted so the
/// correlated subquery sees the complete artifact set. Ordering by `archived_at`
/// (tiebreak: `id`) matches the display ordering used in listings.
pub fn refresh_entry_cached_bytes(conn: &Connection, entry_id: i64) -> Result<()> {
let cached: i64 = conn.query_row(
"SELECT COALESCE(SUM(b.byte_size), 0)
FROM entry_artifacts ea
JOIN blobs b ON b.id = ea.blob_id
JOIN archived_entries e ON e.id = ea.entry_id
WHERE ea.entry_id = ?1
AND ea.blob_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM entry_artifacts ea2
JOIN archived_entries e2 ON e2.id = ea2.entry_id
WHERE ea2.blob_id = ea.blob_id
AND (e2.archived_at < e.archived_at
OR (e2.archived_at = e.archived_at AND e2.id < ?1))
)",
[entry_id],
|row| row.get(0),
)?;
conn.execute(
"UPDATE archived_entries SET cached_bytes = ?1 WHERE id = ?2",
params![cached, entry_id],
)?;
Ok(())
}
/// Recomputes `cached_bytes` for entries that shared blobs with `entry_id` and
/// were archived after it.
///
/// Must be called **before** the entry row is deleted so that the shared-blob
/// lookup still works. The inner EXISTS deliberately excludes `entry_id` so each
/// affected entry is recomputed as if that entry no longer exists.
///
/// Intended to be dispatched asynchronously: acknowledge the delete to the user
/// first, then call this on a background thread.
pub fn cascade_cached_bytes_after_delete(conn: &Connection, entry_id: i64) -> Result<()> {
conn.execute(
"UPDATE archived_entries
SET cached_bytes = (
SELECT COALESCE(SUM(b.byte_size), 0)
FROM entry_artifacts ea
JOIN blobs b ON b.id = ea.blob_id
WHERE ea.entry_id = archived_entries.id
AND ea.blob_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM entry_artifacts ea3
JOIN archived_entries e3 ON e3.id = ea3.entry_id
WHERE ea3.blob_id = ea.blob_id
AND e3.id != ?1
AND (e3.archived_at < archived_entries.archived_at
OR (e3.archived_at = archived_entries.archived_at
AND e3.id < archived_entries.id))
)
)
WHERE id IN (
SELECT DISTINCT ea2.entry_id
FROM entry_artifacts ea_del
JOIN entry_artifacts ea2 ON ea2.blob_id = ea_del.blob_id
JOIN archived_entries e_del ON e_del.id = ea_del.entry_id
JOIN archived_entries e2 ON e2.id = ea2.entry_id
WHERE ea_del.entry_id = ?1
AND ea2.entry_id != ?1
AND (e2.archived_at > e_del.archived_at
OR (e2.archived_at = e_del.archived_at AND e2.id > ?1))
)",
[entry_id],
)?;
Ok(())
}
pub fn upsert_blob(conn: &Connection, blob: &BlobRecord) -> Result<i64> {
conn.execute(
"INSERT OR IGNORE INTO blobs (