1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 03:05:32 +02:00

Compare commits

..

No commits in common. "1ff91956a1f9b13b72a021d1c42edf03c0243fa2" and "c85cce579b0dc417c2192f4c44aa448f7a4a7e00" have entirely different histories.

11 changed files with 31 additions and 180 deletions

View file

@ -28,8 +28,6 @@ pub struct EntrySummary {
pub parent_entry_uid: Option<String>, pub parent_entry_uid: Option<String>,
/// True if a `favicon` artifact exists for this entry. /// True if a `favicon` artifact exists for this entry.
pub has_favicon: bool, pub has_favicon: bool,
/// Bytes of blobs already on disk from an earlier entry (precomputed at capture time).
pub cached_bytes: i64,
} }
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
@ -208,8 +206,7 @@ pub fn list_root_entries(conn: &rusqlite::Connection, caller_bits: u32) -> Resul
COUNT(ea.id) AS artifact_count, COUNT(ea.id) AS artifact_count,
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
NULL AS parent_entry_uid, NULL AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon
e.cached_bytes
FROM archived_entries e FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -241,7 +238,6 @@ pub fn list_root_entries(conn: &rusqlite::Connection, caller_bits: u32) -> Resul
total_artifact_bytes: row.get(8)?, total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?, parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0, has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
}) })
})? })?
.collect::<rusqlite::Result<Vec<_>>>()?; .collect::<rusqlite::Result<Vec<_>>>()?;
@ -429,7 +425,6 @@ pub fn list_entries_for_collection(
total_artifact_bytes: row.get(8)?, total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?, parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0, has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
}) })
})? })?
.collect::<rusqlite::Result<Vec<_>>>()?; .collect::<rusqlite::Result<Vec<_>>>()?;
@ -544,8 +539,7 @@ const ENTRY_SELECT_COLS: &str =
e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, \ e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, \
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \ COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, \
parent.entry_uid AS parent_entry_uid, \ parent.entry_uid AS parent_entry_uid, \
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, \ EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon";
e.cached_bytes";
const ENTRY_FROM_JOINS: &str = const ENTRY_FROM_JOINS: &str =
"FROM archived_entries e \ "FROM archived_entries e \
@ -655,7 +649,6 @@ pub fn search_entries(
total_artifact_bytes: row.get(8)?, total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?, parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0, has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
}) })
})? })?
.collect::<rusqlite::Result<Vec<_>>>()?; .collect::<rusqlite::Result<Vec<_>>>()?;
@ -817,8 +810,7 @@ pub fn entries_for_tag(
e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count, e.visibility, si.canonical_url, COUNT(ea.id) AS artifact_count,
COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes, COALESCE(SUM(b.byte_size), 0) AS total_artifact_bytes,
parent.entry_uid AS parent_entry_uid, parent.entry_uid AS parent_entry_uid,
EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon, EXISTS(SELECT 1 FROM entry_artifacts fav WHERE fav.entry_id = e.id AND fav.artifact_role = 'favicon') AS has_favicon
e.cached_bytes
FROM archived_entries e FROM archived_entries e
JOIN source_identities si ON si.id = e.source_identity_id JOIN source_identities si ON si.id = e.source_identity_id
LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id LEFT JOIN entry_artifacts ea ON ea.entry_id = e.id
@ -843,7 +835,6 @@ pub fn entries_for_tag(
total_artifact_bytes: row.get(8)?, total_artifact_bytes: row.get(8)?,
parent_entry_uid: row.get(9)?, parent_entry_uid: row.get(9)?,
has_favicon: row.get::<_, i64>(10)? != 0, has_favicon: row.get::<_, i64>(10)? != 0,
cached_bytes: row.get(11)?,
}) })
})? })?
.collect::<rusqlite::Result<Vec<_>>>()?; .collect::<rusqlite::Result<Vec<_>>>()?;

View file

@ -751,7 +751,7 @@ pub fn perform_capture(
let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp)); let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
} }
let entry = record_media_entry( record_media_entry(
&conn, &conn,
store_path, store_path,
user_id, user_id,
@ -765,7 +765,6 @@ pub fn perform_capture(
byte_size, byte_size,
title_hint, title_hint,
)?; )?;
database::refresh_entry_cached_bytes(&conn, entry.id)?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult { return Ok(CaptureResult {
run_uid: run.run_uid.clone(), run_uid: run.run_uid.clone(),
@ -905,9 +904,6 @@ pub fn perform_capture(
} }
} }
// 7. Store how many bytes this entry gets "for free" from earlier entries.
database::refresh_entry_cached_bytes(&conn, entry.id)?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult { return Ok(CaptureResult {
run_uid: run.run_uid.clone(), run_uid: run.run_uid.clone(),
@ -946,7 +942,7 @@ pub fn perform_capture(
&timestamp, &timestamp,
) { ) {
Ok(_) => { Ok(_) => {
let tweet_entry = record_tweet_entry( record_tweet_entry(
&conn, &conn,
store_path, store_path,
user_id, user_id,
@ -956,7 +952,6 @@ pub fn perform_capture(
source, source,
&tweet_id, &tweet_id,
)?; )?;
database::refresh_entry_cached_bytes(&conn, tweet_entry.id)?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
return Ok(CaptureResult { return Ok(CaptureResult {
run_uid: run.run_uid.clone(), run_uid: run.run_uid.clone(),
@ -1047,7 +1042,7 @@ pub fn perform_capture(
let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp)); let _ = fs::remove_dir_all(store_path.join("temp").join(&timestamp));
} }
let media_entry = record_media_entry( record_media_entry(
&conn, &conn,
store_path, store_path,
user_id, user_id,
@ -1061,7 +1056,6 @@ pub fn perform_capture(
byte_size, byte_size,
None, // title — populated in a later task None, // title — populated in a later task
)?; )?;
database::refresh_entry_cached_bytes(&conn, media_entry.id)?;
database::finish_archive_run(&conn, run.id)?; database::finish_archive_run(&conn, run.id)?;
Ok(CaptureResult { Ok(CaptureResult {

View file

@ -251,8 +251,7 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> {
structured_root_relpath TEXT NOT NULL, structured_root_relpath TEXT NOT NULL,
representation_kind TEXT NOT NULL, representation_kind TEXT NOT NULL,
source_metadata_json TEXT NOT NULL DEFAULT '{}', source_metadata_json TEXT NOT NULL DEFAULT '{}',
display_metadata_json TEXT, display_metadata_json TEXT
cached_bytes INTEGER NOT NULL DEFAULT 0
); );
CREATE TABLE IF NOT EXISTS blobs ( CREATE TABLE IF NOT EXISTS blobs (
@ -351,38 +350,6 @@ pub fn initialize_schema(conn: &Connection) -> Result<()> {
FROM archived_entries ae; FROM archived_entries ae;
"#, "#,
)?; )?;
// Migration: add cached_bytes column to existing databases.
// New databases already have it from the DDL above; the column check is
// the idiomatic SQLite way to run a migration exactly once.
let column_exists: bool = conn.query_row(
"SELECT COUNT(*) FROM pragma_table_info('archived_entries') WHERE name = 'cached_bytes'",
[],
|row| row.get::<_, i64>(0),
)? > 0;
if !column_exists {
conn.execute_batch(
"ALTER TABLE archived_entries ADD COLUMN cached_bytes INTEGER NOT NULL DEFAULT 0;
UPDATE archived_entries
SET cached_bytes = (
SELECT COALESCE(SUM(b.byte_size), 0)
FROM entry_artifacts ea
JOIN blobs b ON b.id = ea.blob_id
WHERE ea.entry_id = archived_entries.id
AND ea.blob_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM entry_artifacts ea2
JOIN archived_entries e2 ON e2.id = ea2.entry_id
WHERE ea2.blob_id = ea.blob_id
AND (e2.archived_at < archived_entries.archived_at
OR (e2.archived_at = archived_entries.archived_at
AND e2.id < archived_entries.id))
)
);",
)?;
}
Ok(()) Ok(())
} }
@ -1224,82 +1191,6 @@ pub fn upsert_source_identity(
Ok(id) Ok(id)
} }
/// Computes and stores `cached_bytes` for a single entry.
///
/// Must be called after all artifacts for the entry have been inserted so the
/// correlated subquery sees the complete artifact set. Ordering by `archived_at`
/// (tiebreak: `id`) matches the display ordering used in listings.
pub fn refresh_entry_cached_bytes(conn: &Connection, entry_id: i64) -> Result<()> {
let cached: i64 = conn.query_row(
"SELECT COALESCE(SUM(b.byte_size), 0)
FROM entry_artifacts ea
JOIN blobs b ON b.id = ea.blob_id
JOIN archived_entries e ON e.id = ea.entry_id
WHERE ea.entry_id = ?1
AND ea.blob_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM entry_artifacts ea2
JOIN archived_entries e2 ON e2.id = ea2.entry_id
WHERE ea2.blob_id = ea.blob_id
AND (e2.archived_at < e.archived_at
OR (e2.archived_at = e.archived_at AND e2.id < ?1))
)",
[entry_id],
|row| row.get(0),
)?;
conn.execute(
"UPDATE archived_entries SET cached_bytes = ?1 WHERE id = ?2",
params![cached, entry_id],
)?;
Ok(())
}
/// Recomputes `cached_bytes` for entries that shared blobs with `entry_id` and
/// were archived after it.
///
/// Must be called **before** the entry row is deleted so that the shared-blob
/// lookup still works. The inner EXISTS deliberately excludes `entry_id` so each
/// affected entry is recomputed as if that entry no longer exists.
///
/// Intended to be dispatched asynchronously: acknowledge the delete to the user
/// first, then call this on a background thread.
pub fn cascade_cached_bytes_after_delete(conn: &Connection, entry_id: i64) -> Result<()> {
conn.execute(
"UPDATE archived_entries
SET cached_bytes = (
SELECT COALESCE(SUM(b.byte_size), 0)
FROM entry_artifacts ea
JOIN blobs b ON b.id = ea.blob_id
WHERE ea.entry_id = archived_entries.id
AND ea.blob_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM entry_artifacts ea3
JOIN archived_entries e3 ON e3.id = ea3.entry_id
WHERE ea3.blob_id = ea.blob_id
AND e3.id != ?1
AND (e3.archived_at < archived_entries.archived_at
OR (e3.archived_at = archived_entries.archived_at
AND e3.id < archived_entries.id))
)
)
WHERE id IN (
SELECT DISTINCT ea2.entry_id
FROM entry_artifacts ea_del
JOIN entry_artifacts ea2 ON ea2.blob_id = ea_del.blob_id
JOIN archived_entries e_del ON e_del.id = ea_del.entry_id
JOIN archived_entries e2 ON e2.id = ea2.entry_id
WHERE ea_del.entry_id = ?1
AND ea2.entry_id != ?1
AND (e2.archived_at > e_del.archived_at
OR (e2.archived_at = e_del.archived_at AND e2.id > ?1))
)",
[entry_id],
)?;
Ok(())
}
pub fn upsert_blob(conn: &Connection, blob: &BlobRecord) -> Result<i64> { pub fn upsert_blob(conn: &Connection, blob: &BlobRecord) -> Result<i64> {
conn.execute( conn.execute(
"INSERT OR IGNORE INTO blobs ( "INSERT OR IGNORE INTO blobs (

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,8 +4,8 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title> <title>Archivr</title>
<script type="module" crossorigin src="/assets/index-DnX_j2fa.js"></script> <script type="module" crossorigin src="/assets/index-BEbGvi2K.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DXhicc6w.css"> <link rel="stylesheet" crossorigin href="/assets/index-MrdP6h9x.css">
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

View file

@ -2,11 +2,11 @@
An open-source self-hosted archiving tool. Work in progress. An open-source self-hosted archiving tool. Work in progress.
- [x] Archiving ## Milestones
- [ ] Archiving
- [x] Archiving media files from social media platforms - [x] Archiving media files from social media platforms
- [x] YouTube Videos - [x] YouTube Videos
- [x] YouTube Playlists
- [x] YouTube Channels
- [x] Twitter Videos - [x] Twitter Videos
- [x] Instagram - [x] Instagram
- [x] Facebook - [x] Facebook
@ -27,23 +27,14 @@ An open-source self-hosted archiving tool. Work in progress.
- [ ] Gmail - [ ] Gmail
- [ ] Outlook - [ ] Outlook
- [ ] Yahoo Mail - [ ] Yahoo Mail
- [x] Management - [ ] Management
- [x] Deduplication - [ ] Deduplication
- [x] Tagging system - [ ] Tagging system
- [x] Search functionality - [ ] Search functionality
- [ ] Categorization - [ ] Categorization
- [x] Metadata extraction and storage - [ ] Metadata extraction and storage
- [x] User Interface - [ ] User Interface
- [x] Web-based UI - [ ] Web-based UI
- [x] Authentication and login
- [x] Archive setup
- [x] Browse and view entries
- [x] Tag management and filtering
- [x] Search entries
- [x] View archive runs
- [x] Capture dialog
- [x] User settings and API tokens
- [x] Admin panel
- [ ] Backup and Sync - [ ] Backup and Sync
- [ ] Cloud backup (AWS S3, Google Cloud Storage) - [ ] Cloud backup (AWS S3, Google Cloud Storage)
- [ ] Local backup - [ ] Local backup

View file

@ -241,7 +241,7 @@ export default function App() {
</div> </div>
)} )}
{view === 'archive' && ( {view === 'archive' && (
<EntriesView <EntriesView
entries={entries} entries={entries}
selectedEntryUid={selectedEntryUid} selectedEntryUid={selectedEntryUid}
onSelectEntry={selectEntry} onSelectEntry={selectEntry}

View file

@ -39,14 +39,7 @@ export default function EntryRow({ entry, archiveId, isSelected, onSelect }) {
<div className="col-type"> <div className="col-type">
<span className="type-pill">{valueText(entry.entity_kind)}</span> <span className="type-pill">{valueText(entry.entity_kind)}</span>
</div> </div>
<div className="col-size"> <div className="col-size">{formatBytes(entry.total_artifact_bytes)}</div>
<span className="size-total">{formatBytes(entry.total_artifact_bytes)}</span>
{entry.cached_bytes > 0 && entry.total_artifact_bytes > 0 && (
<span className="size-cached-pct" title={`${formatBytes(entry.cached_bytes)} already on disk from an earlier entry`}>
{Math.round(entry.cached_bytes / entry.total_artifact_bytes * 100)}% cached
</span>
)}
</div>
<div className="url-cell col-url">{valueText(entry.original_url)}</div> <div className="url-cell col-url">{valueText(entry.original_url)}</div>
</div> </div>
); );

View file

@ -310,16 +310,7 @@ select {
.col-added { width: 162px; color: var(--muted); } .col-added { width: 162px; color: var(--muted); }
.col-title { flex: 1 1 0; min-width: 0; overflow: hidden; display: flex; align-items: center; gap: 0.42em; } .col-title { flex: 1 1 0; min-width: 0; overflow: hidden; display: flex; align-items: center; gap: 0.42em; }
.col-type { width: 116px; } .col-type { width: 116px; }
.col-size { width: 100px; display: flex; flex-direction: column; justify-content: center; gap: 1px; } .col-size { width: 96px; }
.size-total { font-variant-numeric: tabular-nums; }
.size-cached-pct {
font-size: 10px;
color: var(--muted);
opacity: 0.8;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.col-url { flex: 0 0 30%; min-width: 0; overflow: hidden; } .col-url { flex: 0 0 30%; min-width: 0; overflow: hidden; }
.entry-header-row > div:first-child, .entry-header-row > div:first-child,