1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-22 11:15:41 +02:00

feat: Reader mode via Mozilla Readability.js

Adds an opt-in 'Reader mode' advanced option to the capture dialog.
When enabled, Readability.js is injected as a browser script during
SingleFile capture; it fires on single-file-on-before-capture-start,
replaces the page body with the distilled article content, injects a
clean typographic stylesheet, and adds a header with title/byline/site.
Falls back silently if Readability fails (e.g. non-article pages).

- vendor/readability/Readability.js  Apache 2.0, Mozilla, v0.6.0
- singlefile.rs: embed READABILITY_JS + READER_MODE_WRAPPER_JS via
  include_str!; write both to temp dir when reader_mode is true;
  base_single_file_cmd now accepts &[&Path] for multiple --browser-script
- capture.rs: CaptureConfig.reader_mode: bool
- routes.rs: CaptureBody.reader_mode: Option<bool> (defaults false)
- api.js: submitCapture passes reader_mode in payload
- CaptureDialog.jsx: Reader mode toggle in Advanced options (off by default)
This commit is contained in:
TheGeneralist 2026-07-07 20:04:03 +02:00
parent 28c62ddc5a
commit 596a6d1c89
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
9 changed files with 2967 additions and 74 deletions

View file

@ -78,10 +78,9 @@ impl PlatformMetadata {
pub struct CaptureConfig {
pub cookie_rules: Vec<database::CookieRule>,
/// Override for uBlock Origin Lite during WebPage captures.
/// `None` = derive from `ARCHIVR_UBLOCK` env var (existing behaviour).
/// `Some(false)` = disabled by admin/user choice for this capture.
/// `Some(true)` = explicitly enabled (still needs `ARCHIVR_UBLOCK_EXT` to be valid).
pub ublock_enabled: Option<bool>,
/// Apply Mozilla Readability to distil the page to article content before archiving.
pub reader_mode: bool,
}
/// Resolves which cookies apply to `url` by evaluating all rules in ordinal order.
@ -1015,7 +1014,7 @@ pub fn perform_capture(
// Source: web page — archive as a self-contained HTML snapshot via single-file-cli
if source == Source::WebPage {
match downloader::singlefile::save(locator, store_path, &timestamp, &cookies, config.ublock_enabled) {
match downloader::singlefile::save(locator, store_path, &timestamp, &cookies, config.ublock_enabled, config.reader_mode) {
Ok(result) => {
let file_extension = ".html".to_string();
let temp_html = store_path

View file

@ -14,6 +14,51 @@ use std::{
use crate::downloader::cookies::{domain_from_url, write_netscape_cookie_file};
use crate::hash::hash_file;
/// Mozilla Readability.js (Apache 2.0) — embedded at compile time so captures
/// don't need it on PATH. Path is relative to this source file.
const READABILITY_JS: &str =
include_str!("../../../../vendor/readability/Readability.js");
/// Single-file user script that applies Readability just before the page is
/// serialised. Fires on `single-file-on-before-capture-start`.
const READER_MODE_WRAPPER_JS: &str = r#"
addEventListener('single-file-on-before-capture-start', function() {
try {
if (typeof Readability === 'undefined') return;
var article = new Readability(document.cloneNode(true)).parse();
if (!article || !article.content || article.content.length < 100) return;
document.body.innerHTML = article.content;
if (article.title) document.title = article.title;
var hdr = document.createElement('header');
hdr.innerHTML =
'<h1 style="margin:0 0 .3em;font-family:-apple-system,sans-serif">' +
(article.title || '') + '</h1>' +
(article.byline
? '<p style="margin:0;color:#666;font-size:14px">' + article.byline + '</p>'
: '') +
(article.siteName
? '<p style="margin:.2em 0 0;color:#999;font-size:12px">' + article.siteName + '</p>'
: '');
hdr.style.cssText = 'margin-bottom:2em;padding-bottom:1em;border-bottom:1px solid #ddd';
document.body.insertBefore(hdr, document.body.firstChild);
var style = document.createElement('style');
style.textContent = [
'body{max-width:680px;margin:40px auto;padding:0 24px;',
'font-family:Georgia,"Times New Roman",serif;font-size:18px;',
'line-height:1.75;color:#1a1a1a;background:#fafaf8}',
'h1,h2,h3,h4,h5,h6{font-family:-apple-system,BlinkMacSystemFont,sans-serif;',
'line-height:1.3;margin-top:1.5em}',
'img,figure,video{max-width:100%;height:auto;display:block;margin:1em 0}',
'a{color:#0055cc}',
'pre{background:#f4f4f4;padding:1em;border-radius:4px;overflow-x:auto;font-size:14px}',
'code{background:#f4f4f4;padding:.1em .3em;border-radius:3px;font-size:14px}',
'blockquote{border-left:3px solid #ccc;margin:1em 0;padding-left:1.2em;color:#555}',
].join('');
document.head.appendChild(style);
} catch (e) { /* non-fatal: fall back to original page */ }
});
"#;
/// Result of archiving a web page with single-file.
#[derive(Debug)]
pub struct SaveResult {
@ -44,6 +89,7 @@ pub fn save(
timestamp: &str,
cookies: &HashMap<String, String>,
ublock_enabled_override: Option<bool>,
reader_mode: bool,
) -> Result<SaveResult> {
let single_file =
env::var("ARCHIVR_SINGLE_FILE").unwrap_or_else(|_| "single-file".to_string());
@ -57,6 +103,7 @@ pub fn save(
&chrome,
cookies,
ublock_ext.as_deref(),
reader_mode,
)?;
result.ublock_skipped = ublock_skipped;
Ok(result)
@ -121,20 +168,21 @@ fn save_with(
chrome: &str,
cookies: &HashMap<String, String>,
ublock_ext: Option<&Path>,
reader_mode: bool,
) -> Result<SaveResult> {
let temp_dir = store_path.join("temp").join(timestamp);
std::fs::create_dir_all(&temp_dir).context("failed to create temp dir")?;
let out_file = temp_dir.join(format!("{timestamp}.html"));
// User script that strips <script> elements from the live DOM just before
// SingleFile serializes it. Lets scripts execute during capture (so
// Mandatory user script: strips <script> elements from the live DOM just
// before SingleFile serializes it. Lets scripts execute during capture (so
// JS-applied CSS is present) without leaving data:-URL ES modules in the
// saved file that would cause "base scheme isn't hierarchical" errors.
// JSON-LD structured data is kept.
let user_script_path = temp_dir.join("sf-strip-scripts.js");
let strip_scripts_path = temp_dir.join("sf-strip-scripts.js");
std::fs::write(
&user_script_path,
&strip_scripts_path,
"addEventListener('single-file-on-before-capture-start',()=>{\
document.querySelectorAll('script:not([type=\"application/ld+json\"])')\
.forEach(el=>el.remove());\
@ -142,8 +190,21 @@ fn save_with(
)
.context("failed to write single-file user script")?;
// Isolated Chrome profile directory; keeps each capture's browser state
// separate and cleans up with the rest of the temp dir.
// Reader-mode scripts: Readability.js + wrapper. Written to temp dir and
// passed as additional --browser-script args when reader mode is enabled.
let mut extra_browser_scripts: Vec<PathBuf> = Vec::new();
if reader_mode {
let readability_path = temp_dir.join("sf-readability.js");
std::fs::write(&readability_path, READABILITY_JS)
.context("failed to write Readability.js script")?;
let wrapper_path = temp_dir.join("sf-reader-mode.js");
std::fs::write(&wrapper_path, READER_MODE_WRAPPER_JS)
.context("failed to write reader-mode wrapper script")?;
extra_browser_scripts.push(readability_path);
extra_browser_scripts.push(wrapper_path);
}
// Isolated Chrome profile directory.
let chrome_data_dir = temp_dir.join("chrome-data");
// Extra Chrome flags from the environment (e.g. "--no-sandbox" in Docker).
@ -198,12 +259,15 @@ fn save_with(
bail!("Chrome did not become ready on port {port} within 10 s");
}
// Build scripts list: strip-scripts first, then any reader-mode scripts.
let mut scripts: Vec<&Path> = vec![strip_scripts_path.as_path()];
scripts.extend(extra_browser_scripts.iter().map(|p| p.as_path()));
let out = run_single_file_with_server(
url,
&out_file,
single_file,
port,
&user_script_path,
&scripts,
cookie_file.as_deref(),
);
@ -229,13 +293,15 @@ fn save_with(
.collect();
let browser_args = format!("[{}]", quoted.join(","));
let mut scripts: Vec<&Path> = vec![strip_scripts_path.as_path()];
scripts.extend(extra_browser_scripts.iter().map(|p| p.as_path()));
run_single_file_standalone(
url,
&out_file,
single_file,
chrome,
&browser_args,
&user_script_path,
&scripts,
cookie_file.as_deref(),
)
.with_context(|| format!("failed to spawn single-file ({single_file})"))?
@ -323,10 +389,10 @@ fn run_single_file_with_server(
out_file: &Path,
single_file: &str,
port: u16,
user_script_path: &Path,
scripts: &[&Path],
cookie_file: Option<&Path>,
) -> std::io::Result<std::process::Output> {
let mut cmd = base_single_file_cmd(url, out_file, single_file, user_script_path, cookie_file);
let mut cmd = base_single_file_cmd(url, out_file, single_file, scripts, cookie_file);
cmd.arg(format!("--browser-server=http://127.0.0.1:{port}"));
cmd.output()
}
@ -337,11 +403,11 @@ fn run_single_file_standalone(
out_file: &Path,
single_file: &str,
chrome: &str,
browser_args: &str, // JSON array string, e.g. `["--disable-web-security",...]`
user_script_path: &Path,
browser_args: &str,
scripts: &[&Path],
cookie_file: Option<&Path>,
) -> std::io::Result<std::process::Output> {
let mut cmd = base_single_file_cmd(url, out_file, single_file, user_script_path, cookie_file);
let mut cmd = base_single_file_cmd(url, out_file, single_file, scripts, cookie_file);
cmd.arg(format!("--browser-executable-path={chrome}"))
.arg("--browser-headless")
.arg(format!("--browser-args={browser_args}"));
@ -349,32 +415,28 @@ fn run_single_file_standalone(
}
/// Builds a `Command` with the single-file args that are the same regardless
/// of how Chrome is started.
/// of how Chrome is started. Passes each script as a separate `--browser-script` arg.
fn base_single_file_cmd(
url: &str,
out_file: &Path,
single_file: &str,
user_script_path: &Path,
scripts: &[&Path],
cookie_file: Option<&Path>,
) -> Command {
let mut cmd = Command::new(single_file);
cmd.arg(url)
.arg(out_file)
.arg("--browser-wait-until=networkidle2")
// Extra delay after networkidle2: Cloudflare Fonts injects @font-face
// CSS after HTML parse, so the font hook needs more time to see it.
.arg("--browser-wait-delay=2000")
// Realistic UA: some origins block headless Chrome's default UA string.
.arg("--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36")
// Preserve all CSS.
.arg("--remove-unused-styles=false")
.arg("--remove-alternative-medias=false")
// Allow scripts during capture so JS-applied classes exist in the DOM.
.arg("--block-scripts=false")
.arg(format!("--browser-script={}", user_script_path.display()))
// Preserve fonts.
.arg("--remove-unused-fonts=false")
.arg("--remove-alternative-fonts=false");
for script in scripts {
cmd.arg(format!("--browser-script={}", script.display()));
}
if let Some(cf) = cookie_file {
cmd.arg(format!("--browser-cookies-file={}", cf.display()));
}
@ -516,6 +578,7 @@ mod tests {
"chromium",
&HashMap::new(),
None, // no ublock ext
false, // reader mode off
);
let err = result.unwrap_err();
let msg = format!("{err:#}");