1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 03:05:32 +02:00

feat: Reader mode via Mozilla Readability.js

Adds an opt-in 'Reader mode' advanced option to the capture dialog.
When enabled, Readability.js is injected as a browser script during
SingleFile capture; it fires on single-file-on-before-capture-start,
replaces the page body with the distilled article content, injects a
clean typographic stylesheet, and adds a header with title/byline/site.
Falls back silently if Readability fails (e.g. non-article pages).

- vendor/readability/Readability.js  Apache 2.0, Mozilla, v0.6.0
- singlefile.rs: embed READABILITY_JS + READER_MODE_WRAPPER_JS via
  include_str!; write both to temp dir when reader_mode is true;
  base_single_file_cmd now accepts &[&Path] for multiple --browser-script
- capture.rs: CaptureConfig.reader_mode: bool
- routes.rs: CaptureBody.reader_mode: Option<bool> (defaults false)
- api.js: submitCapture passes reader_mode in payload
- CaptureDialog.jsx: Reader mode toggle in Advanced options (off by default)
This commit is contained in:
TheGeneralist 2026-07-07 20:04:03 +02:00
parent 28c62ddc5a
commit 596a6d1c89
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
9 changed files with 2967 additions and 74 deletions

View file

@ -78,10 +78,9 @@ impl PlatformMetadata {
pub struct CaptureConfig { pub struct CaptureConfig {
pub cookie_rules: Vec<database::CookieRule>, pub cookie_rules: Vec<database::CookieRule>,
/// Override for uBlock Origin Lite during WebPage captures. /// Override for uBlock Origin Lite during WebPage captures.
/// `None` = derive from `ARCHIVR_UBLOCK` env var (existing behaviour).
/// `Some(false)` = disabled by admin/user choice for this capture.
/// `Some(true)` = explicitly enabled (still needs `ARCHIVR_UBLOCK_EXT` to be valid).
pub ublock_enabled: Option<bool>, pub ublock_enabled: Option<bool>,
/// Apply Mozilla Readability to distil the page to article content before archiving.
pub reader_mode: bool,
} }
/// Resolves which cookies apply to `url` by evaluating all rules in ordinal order. /// Resolves which cookies apply to `url` by evaluating all rules in ordinal order.
@ -1015,7 +1014,7 @@ pub fn perform_capture(
// Source: web page — archive as a self-contained HTML snapshot via single-file-cli // Source: web page — archive as a self-contained HTML snapshot via single-file-cli
if source == Source::WebPage { if source == Source::WebPage {
match downloader::singlefile::save(locator, store_path, &timestamp, &cookies, config.ublock_enabled) { match downloader::singlefile::save(locator, store_path, &timestamp, &cookies, config.ublock_enabled, config.reader_mode) {
Ok(result) => { Ok(result) => {
let file_extension = ".html".to_string(); let file_extension = ".html".to_string();
let temp_html = store_path let temp_html = store_path

View file

@ -14,6 +14,51 @@ use std::{
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file}; use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
use crate::hash::hash_file; use crate::hash::hash_file;
/// Mozilla Readability.js (Apache 2.0) — embedded at compile time so captures
/// don't need it on PATH. Path is relative to this source file.
const READABILITY_JS: &str =
include_str!("../../../../vendor/readability/Readability.js");
/// Single-file user script that applies Readability just before the page is
/// serialised. Fires on `single-file-on-before-capture-start`.
const READER_MODE_WRAPPER_JS: &str = r#"
addEventListener('single-file-on-before-capture-start', function() {
try {
if (typeof Readability === 'undefined') return;
var article = new Readability(document.cloneNode(true)).parse();
if (!article || !article.content || article.content.length < 100) return;
document.body.innerHTML = article.content;
if (article.title) document.title = article.title;
var hdr = document.createElement('header');
hdr.innerHTML =
'<h1 style="margin:0 0 .3em;font-family:-apple-system,sans-serif">' +
(article.title || '') + '</h1>' +
(article.byline
? '<p style="margin:0;color:#666;font-size:14px">' + article.byline + '</p>'
: '') +
(article.siteName
? '<p style="margin:.2em 0 0;color:#999;font-size:12px">' + article.siteName + '</p>'
: '');
hdr.style.cssText = 'margin-bottom:2em;padding-bottom:1em;border-bottom:1px solid #ddd';
document.body.insertBefore(hdr, document.body.firstChild);
var style = document.createElement('style');
style.textContent = [
'body{max-width:680px;margin:40px auto;padding:0 24px;',
'font-family:Georgia,"Times New Roman",serif;font-size:18px;',
'line-height:1.75;color:#1a1a1a;background:#fafaf8}',
'h1,h2,h3,h4,h5,h6{font-family:-apple-system,BlinkMacSystemFont,sans-serif;',
'line-height:1.3;margin-top:1.5em}',
'img,figure,video{max-width:100%;height:auto;display:block;margin:1em 0}',
'a{color:#0055cc}',
'pre{background:#f4f4f4;padding:1em;border-radius:4px;overflow-x:auto;font-size:14px}',
'code{background:#f4f4f4;padding:.1em .3em;border-radius:3px;font-size:14px}',
'blockquote{border-left:3px solid #ccc;margin:1em 0;padding-left:1.2em;color:#555}',
].join('');
document.head.appendChild(style);
} catch (e) { /* non-fatal: fall back to original page */ }
});
"#;
/// Result of archiving a web page with single-file. /// Result of archiving a web page with single-file.
#[derive(Debug)] #[derive(Debug)]
pub struct SaveResult { pub struct SaveResult {
@ -44,6 +89,7 @@ pub fn save(
timestamp: &str, timestamp: &str,
cookies: &HashMap<String, String>, cookies: &HashMap<String, String>,
ublock_enabled_override: Option<bool>, ublock_enabled_override: Option<bool>,
reader_mode: bool,
) -> Result<SaveResult> { ) -> Result<SaveResult> {
let single_file = let single_file =
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string()); env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
@ -57,6 +103,7 @@ pub fn save(
&chrome, &chrome,
cookies, cookies,
ublock_ext.as_deref(), ublock_ext.as_deref(),
reader_mode,
)?; )?;
result.ublock_skipped = ublock_skipped; result.ublock_skipped = ublock_skipped;
Ok(result) Ok(result)
@ -121,20 +168,21 @@ fn save_with(
chrome: &str, chrome: &str,
cookies: &HashMap<String, String>, cookies: &HashMap<String, String>,
ublock_ext: Option<&Path>, ublock_ext: Option<&Path>,
reader_mode: bool,
) -> Result<SaveResult> { ) -> Result<SaveResult> {
let temp_dir = store_path.join("temp").join(timestamp); let temp_dir = store_path.join("temp").join(timestamp);
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?; std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
let out_file = temp_dir.join(format!("{timestamp}.html")); let out_file = temp_dir.join(format!("{timestamp}.html"));
// User script that strips <script> elements from the live DOM just before // Mandatory user script: strips <script> elements from the live DOM just
// SingleFile serializes it. Lets scripts execute during capture (so // before SingleFile serializes it. Lets scripts execute during capture (so
// JS-applied CSS is present) without leaving data:-URL ES modules in the // JS-applied CSS is present) without leaving data:-URL ES modules in the
// saved file that would cause "base scheme isn't hierarchical" errors. // saved file that would cause "base scheme isn't hierarchical" errors.
// JSON-LD structured data is kept. // JSON-LD structured data is kept.
let user_script_path = temp_dir.join("sf-strip-scripts.js"); let strip_scripts_path = temp_dir.join("sf-strip-scripts.js");
std::fs::write( std::fs::write(
&user_script_path, &strip_scripts_path,
"addEventListener('single-file-on-before-capture-start',()=>{\ "addEventListener('single-file-on-before-capture-start',()=>{\
document.querySelectorAll('script:not([type=\"application/ld+json\"])')\ document.querySelectorAll('script:not([type=\"application/ld+json\"])')\
.forEach(el=>el.remove());\ .forEach(el=>el.remove());\
@ -142,8 +190,21 @@ fn save_with(
) )
.context("failed to write single-file user script")?; .context("failed to write single-file user script")?;
// Isolated Chrome profile directory; keeps each capture's browser state // Reader-mode scripts: Readability.js + wrapper. Written to temp dir and
// separate and cleans up with the rest of the temp dir. // passed as additional --browser-script args when reader mode is enabled.
let mut extra_browser_scripts: Vec<PathBuf> = Vec::new();
if reader_mode {
let readability_path = temp_dir.join("sf-readability.js");
std::fs::write(&readability_path, READABILITY_JS)
.context("failed to write Readability.js script")?;
let wrapper_path = temp_dir.join("sf-reader-mode.js");
std::fs::write(&wrapper_path, READER_MODE_WRAPPER_JS)
.context("failed to write reader-mode wrapper script")?;
extra_browser_scripts.push(readability_path);
extra_browser_scripts.push(wrapper_path);
}
// Isolated Chrome profile directory.
let chrome_data_dir = temp_dir.join("chrome-data"); let chrome_data_dir = temp_dir.join("chrome-data");
// Extra Chrome flags from the environment (e.g. "--no-sandbox" in Docker). // Extra Chrome flags from the environment (e.g. "--no-sandbox" in Docker).
@ -198,12 +259,15 @@ fn save_with(
bail!("Chrome did not become ready on port {port} within 10 s"); bail!("Chrome did not become ready on port {port} within 10 s");
} }
// Build scripts list: strip-scripts first, then any reader-mode scripts.
let mut scripts: Vec<&Path> = vec![strip_scripts_path.as_path()];
scripts.extend(extra_browser_scripts.iter().map(|p| p.as_path()));
let out = run_single_file_with_server( let out = run_single_file_with_server(
url, url,
&out_file, &out_file,
single_file, single_file,
port, port,
&user_script_path, &scripts,
cookie_file.as_deref(), cookie_file.as_deref(),
); );
@ -229,13 +293,15 @@ fn save_with(
.collect(); .collect();
let browser_args = format!("[{}]", quoted.join(",")); let browser_args = format!("[{}]", quoted.join(","));
let mut scripts: Vec<&Path> = vec![strip_scripts_path.as_path()];
scripts.extend(extra_browser_scripts.iter().map(|p| p.as_path()));
run_single_file_standalone( run_single_file_standalone(
url, url,
&out_file, &out_file,
single_file, single_file,
chrome, chrome,
&browser_args, &browser_args,
&user_script_path, &scripts,
cookie_file.as_deref(), cookie_file.as_deref(),
) )
.with_context(|| format!("failed to spawn single-file ({single_file})"))? .with_context(|| format!("failed to spawn single-file ({single_file})"))?
@ -323,10 +389,10 @@ fn run_single_file_with_server(
out_file: &Path, out_file: &Path,
single_file: &str, single_file: &str,
port: u16, port: u16,
user_script_path: &Path, scripts: &[&Path],
cookie_file: Option<&Path>, cookie_file: Option<&Path>,
) -> std::io::Result<std::process::Output> { ) -> std::io::Result<std::process::Output> {
let mut cmd = base_single_file_cmd(url, out_file, single_file, user_script_path, cookie_file); let mut cmd = base_single_file_cmd(url, out_file, single_file, scripts, cookie_file);
cmd.arg(format!("--browser-server=http://127.0.0.1:{port}")); cmd.arg(format!("--browser-server=http://127.0.0.1:{port}"));
cmd.output() cmd.output()
} }
@ -337,11 +403,11 @@ fn run_single_file_standalone(
out_file: &Path, out_file: &Path,
single_file: &str, single_file: &str,
chrome: &str, chrome: &str,
browser_args: &str, // JSON array string, e.g. `["--disable-web-security",...]` browser_args: &str,
user_script_path: &Path, scripts: &[&Path],
cookie_file: Option<&Path>, cookie_file: Option<&Path>,
) -> std::io::Result<std::process::Output> { ) -> std::io::Result<std::process::Output> {
let mut cmd = base_single_file_cmd(url, out_file, single_file, user_script_path, cookie_file); let mut cmd = base_single_file_cmd(url, out_file, single_file, scripts, cookie_file);
cmd.arg(format!("--browser-executable-path={chrome}")) cmd.arg(format!("--browser-executable-path={chrome}"))
.arg("--browser-headless") .arg("--browser-headless")
.arg(format!("--browser-args={browser_args}")); .arg(format!("--browser-args={browser_args}"));
@ -349,32 +415,28 @@ fn run_single_file_standalone(
} }
/// Builds a `Command` with the single-file args that are the same regardless /// Builds a `Command` with the single-file args that are the same regardless
/// of how Chrome is started. /// of how Chrome is started. Passes each script as a separate `--browser-script` arg.
fn base_single_file_cmd( fn base_single_file_cmd(
url: &str, url: &str,
out_file: &Path, out_file: &Path,
single_file: &str, single_file: &str,
user_script_path: &Path, scripts: &[&Path],
cookie_file: Option<&Path>, cookie_file: Option<&Path>,
) -> Command { ) -> Command {
let mut cmd = Command::new(single_file); let mut cmd = Command::new(single_file);
cmd.arg(url) cmd.arg(url)
.arg(out_file) .arg(out_file)
.arg("--browser-wait-until=networkidle2") .arg("--browser-wait-until=networkidle2")
// Extra delay after networkidle2: Cloudflare Fonts injects @font-face
// CSS after HTML parse, so the font hook needs more time to see it.
.arg("--browser-wait-delay=2000") .arg("--browser-wait-delay=2000")
// Realistic UA: some origins block headless Chrome's default UA string.
.arg("--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36") .arg("--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36")
// Preserve all CSS.
.arg("--remove-unused-styles=false") .arg("--remove-unused-styles=false")
.arg("--remove-alternative-medias=false") .arg("--remove-alternative-medias=false")
// Allow scripts during capture so JS-applied classes exist in the DOM.
.arg("--block-scripts=false") .arg("--block-scripts=false")
.arg(format!("--browser-script={}", user_script_path.display()))
// Preserve fonts.
.arg("--remove-unused-fonts=false") .arg("--remove-unused-fonts=false")
.arg("--remove-alternative-fonts=false"); .arg("--remove-alternative-fonts=false");
for script in scripts {
cmd.arg(format!("--browser-script={}", script.display()));
}
if let Some(cf) = cookie_file { if let Some(cf) = cookie_file {
cmd.arg(format!("--browser-cookies-file={}", cf.display())); cmd.arg(format!("--browser-cookies-file={}", cf.display()));
} }
@ -516,6 +578,7 @@ mod tests {
"chromium", "chromium",
&HashMap::new(), &HashMap::new(),
None, // no ublock ext None, // no ublock ext
false, // reader mode off
); );
let err = result.unwrap_err(); let err = result.unwrap_err();
let msg = format!("{err:#}"); let msg = format!("{err:#}");

View file

@ -650,11 +650,10 @@ async fn delete_entry_handler(
#[derive(Debug, serde::Deserialize)] #[derive(Debug, serde::Deserialize)]
struct CaptureBody { struct CaptureBody {
locator: String, locator: String,
/// Optional quality cap for yt-dlp sources: `"best"` or any `"NNNp"` string
/// (e.g. `"1080p"`, `"720p"`, `"2160p"`). Absent or `"best"` → highest available.
quality: Option<String>, quality: Option<String>,
/// Per-capture uBlock override. Absent → use global instance setting.
ublock_enabled: Option<bool>, ublock_enabled: Option<bool>,
/// Distil to article content via Readability before archiving. Absent = false.
reader_mode: Option<bool>,
} }
#[derive(Debug, serde::Deserialize)] #[derive(Debug, serde::Deserialize)]
@ -754,6 +753,7 @@ async fn capture_handler(
let capture_config = capture::CaptureConfig { let capture_config = capture::CaptureConfig {
cookie_rules, cookie_rules,
ublock_enabled: Some(effective_ublock), ublock_enabled: Some(effective_ublock),
reader_mode: body.reader_mode.unwrap_or(false),
}; };
// Spawn background capture. // Spawn background capture.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -4,7 +4,7 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title> <title>Archivr</title>
<script type="module" crossorigin src="/assets/index-BB_JhtCc.js"></script> <script type="module" crossorigin src="/assets/index-Cxnyn8_x.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-C0BIEdow.css"> <link rel="stylesheet" crossorigin href="/assets/index-C0BIEdow.css">
</head> </head>
<body> <body>

View file

@ -98,9 +98,10 @@ export async function fetchTags(archiveId) {
export async function submitCapture(archiveId, locator, quality = null, extensions = null) { export async function submitCapture(archiveId, locator, quality = null, extensions = null) {
const payload = { locator } const payload = { locator }
if (quality && quality !== 'best') payload.quality = quality if (quality && quality !== 'best') payload.quality = quality
// extensions: { ublock_enabled?: bool } — per-capture overrides // extensions: { ublock_enabled?: bool, reader_mode?: bool } — per-capture overrides
if (extensions && typeof extensions.ublock_enabled === 'boolean') { if (extensions) {
payload.ublock_enabled = extensions.ublock_enabled if (typeof extensions.ublock_enabled === 'boolean') payload.ublock_enabled = extensions.ublock_enabled
if (typeof extensions.reader_mode === 'boolean') payload.reader_mode = extensions.reader_mode
} }
const res = await fetch(`/api/archives/${archiveId}/captures`, { const res = await fetch(`/api/archives/${archiveId}/captures`, {
method: "POST", method: "POST",

View file

@ -128,6 +128,8 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
// Effective uBlock for this session // Effective uBlock for this session
const ublockEnabled = ublockOverride !== null ? ublockOverride : (globalUblock ?? true) const ublockEnabled = ublockOverride !== null ? ublockOverride : (globalUblock ?? true)
// Reader mode: off by default, per-session only
const [readerMode, setReaderMode] = useState(false)
// On mount: clean up old single-locator sessionStorage keys; reconnect running jobs // On mount: clean up old single-locator sessionStorage keys; reconnect running jobs
useEffect(() => { useEffect(() => {
@ -240,7 +242,7 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
const qual = item.quality || 'best' const qual = item.quality || 'best'
setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it)) setItems(prev => prev.map(it => it.id === item.id ? { ...it, status: 'submitting', error: null } : it))
try { try {
const extensions = { ublock_enabled: ublockEnabled } const extensions = { ublock_enabled: ublockEnabled, reader_mode: readerMode }
const job = await submitCapture(aid, loc, qual, extensions) const job = await submitCapture(aid, loc, qual, extensions)
setItems(prev => prev.map(it => setItems(prev => prev.map(it =>
it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it it.id === item.id ? { ...it, status: 'running', jobUid: job.job_uid, archiveId: aid } : it
@ -395,6 +397,22 @@ export default function CaptureDialog({ open, archiveId, onClose, onCaptured, on
<span className="ext-toggle-knob" /> <span className="ext-toggle-knob" />
</button> </button>
</label> </label>
<label className="capture-ext-row" style={{ marginTop: 8 }}>
<span className="capture-ext-label">
<span className="capture-ext-name">Reader mode</span>
<span className="capture-ext-desc">Distil to article text via Readability (off by default)</span>
</span>
<button
type="button"
role="switch"
aria-checked={readerMode}
className={`ext-toggle ext-toggle--sm${readerMode ? ' ext-toggle--on' : ''}`}
onClick={() => setReaderMode(v => !v)}
aria-label="Toggle reader mode for this capture"
>
<span className="ext-toggle-knob" />
</button>
</label>
</div> </div>
)} )}
</div> </div>

2812
vendor/readability/Readability.js vendored Normal file

File diff suppressed because it is too large Load diff