mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-22 03:05:32 +02:00
refactor: add core archive opening APIs
This commit is contained in:
parent
3cf2c9cd3a
commit
5b9a5bcaa4
3 changed files with 199 additions and 65 deletions
|
|
@ -1,5 +1,5 @@
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use archivr_core::{database, downloader, twitter::parse_tweet_id};
|
use archivr_core::{archive, database, downloader, twitter::parse_tweet_id};
|
||||||
use chrono::Local;
|
use chrono::Local;
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
@ -53,19 +53,6 @@ enum Command {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_archive_path() -> Result<Option<PathBuf>> {
|
|
||||||
let mut dir = env::current_dir().context("failed to read current working directory")?;
|
|
||||||
loop {
|
|
||||||
if dir.join(".archivr").is_dir() {
|
|
||||||
return Ok(Some(dir.join(".archivr")));
|
|
||||||
}
|
|
||||||
if !dir.pop() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
|
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
|
||||||
enum Source {
|
enum Source {
|
||||||
YouTubeVideo,
|
YouTubeVideo,
|
||||||
|
|
@ -310,14 +297,6 @@ fn move_temp_to_raw(file: &Path, hash: &str, store_path: &Path) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn initialize_store_directories(store_path: &Path) -> Result<()> {
|
|
||||||
fs::create_dir_all(store_path.join("raw"))?;
|
|
||||||
fs::create_dir_all(store_path.join("raw_tweets"))?;
|
|
||||||
fs::create_dir_all(store_path.join("structured"))?;
|
|
||||||
fs::create_dir_all(store_path.join("temp"))?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result<PathBuf> {
|
fn raw_relative_path_from_hash(hash: &str, file_extension: &str) -> Result<PathBuf> {
|
||||||
let mut chars = hash.chars();
|
let mut chars = hash.chars();
|
||||||
let first_letter = chars.next().context("hash must not be empty")?;
|
let first_letter = chars.next().context("hash must not be empty")?;
|
||||||
|
|
@ -609,7 +588,7 @@ fn main() -> Result<()> {
|
||||||
|
|
||||||
match args.command {
|
match args.command {
|
||||||
Command::Archive { ref path } => {
|
Command::Archive { ref path } => {
|
||||||
let archive_path = match get_archive_path()? {
|
let archive_path = match archive::find_archive_path()? {
|
||||||
Some(path) => path,
|
Some(path) => path,
|
||||||
None => {
|
None => {
|
||||||
eprintln!("Not in an archive. Use 'archivr init' to create one.");
|
eprintln!("Not in an archive. Use 'archivr init' to create one.");
|
||||||
|
|
@ -833,7 +812,7 @@ fn main() -> Result<()> {
|
||||||
name: ref archive_name,
|
name: ref archive_name,
|
||||||
force_with_info_removal,
|
force_with_info_removal,
|
||||||
} => {
|
} => {
|
||||||
let archive_path = Path::new(&archive_path_string).join(".archivr");
|
let archive_parent = Path::new(&archive_path_string);
|
||||||
let store_path = if Path::new(&store_path_string).is_relative() {
|
let store_path = if Path::new(&store_path_string).is_relative() {
|
||||||
env::current_dir()
|
env::current_dir()
|
||||||
.context("failed to read current working directory")?
|
.context("failed to read current working directory")?
|
||||||
|
|
@ -842,47 +821,17 @@ fn main() -> Result<()> {
|
||||||
Path::new(store_path_string).to_path_buf()
|
Path::new(store_path_string).to_path_buf()
|
||||||
};
|
};
|
||||||
|
|
||||||
if archive_path.exists() {
|
let paths = archive::initialize_archive(
|
||||||
if !archive_path.is_dir() {
|
archive_parent,
|
||||||
eprintln!(
|
&store_path,
|
||||||
"Archive path exists and is not a directory: {}",
|
archive_name,
|
||||||
archive_path.display()
|
force_with_info_removal,
|
||||||
);
|
|
||||||
process::exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if force_with_info_removal {
|
|
||||||
fs::remove_dir_all(&archive_path)?;
|
|
||||||
} else if fs::read_dir(&archive_path)?.next().is_some() {
|
|
||||||
eprintln!(
|
|
||||||
"Archive already exists at {} and is not empty. Use --force-with-info-removal to reinitialize.",
|
|
||||||
archive_path.display()
|
|
||||||
);
|
|
||||||
process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if store_path.exists() && !force_with_info_removal {
|
|
||||||
eprintln!("Store path already exists at {}", store_path.display());
|
|
||||||
process::exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
fs::create_dir_all(&archive_path)?;
|
|
||||||
fs::create_dir_all(&store_path)?;
|
|
||||||
fs::write(archive_path.join("name"), archive_name)?;
|
|
||||||
fs::write(
|
|
||||||
archive_path.join("store_path"),
|
|
||||||
store_path
|
|
||||||
.canonicalize()
|
|
||||||
.with_context(|| format!("failed to canonicalize {}", store_path.display()))?
|
|
||||||
.to_str()
|
|
||||||
.context("store path is not valid UTF-8")?,
|
|
||||||
)?;
|
)?;
|
||||||
initialize_store_directories(&store_path)?;
|
|
||||||
let conn = database::open_or_initialize(&archive_path)?;
|
|
||||||
let _ = database::ensure_default_user(&conn)?;
|
|
||||||
|
|
||||||
println!("Initialized empty archive in {}", archive_path.display());
|
println!(
|
||||||
|
"Initialized empty archive in {}",
|
||||||
|
paths.archive_path.display()
|
||||||
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
} // _ => eprintln!("Unknown command: {:?}", args.command),
|
} // _ => eprintln!("Unknown command: {:?}", args.command),
|
||||||
|
|
@ -1254,7 +1203,7 @@ mod tests {
|
||||||
Local::now().format("%Y%m%d%H%M%S%3f")
|
Local::now().format("%Y%m%d%H%M%S%3f")
|
||||||
));
|
));
|
||||||
|
|
||||||
initialize_store_directories(&store_path).unwrap();
|
archive::initialize_store_directories(&store_path).unwrap();
|
||||||
|
|
||||||
assert!(store_path.join("raw").is_dir());
|
assert!(store_path.join("raw").is_dir());
|
||||||
assert!(store_path.join("raw_tweets").is_dir());
|
assert!(store_path.join("raw_tweets").is_dir());
|
||||||
|
|
@ -1272,7 +1221,7 @@ mod tests {
|
||||||
Local::now().format("%Y%m%d%H%M%S%3f")
|
Local::now().format("%Y%m%d%H%M%S%3f")
|
||||||
));
|
));
|
||||||
let _ = fs::remove_dir_all(&store_path);
|
let _ = fs::remove_dir_all(&store_path);
|
||||||
initialize_store_directories(&store_path).unwrap();
|
archive::initialize_store_directories(&store_path).unwrap();
|
||||||
fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap();
|
fs::create_dir_all(store_path.join("raw").join("a").join("b")).unwrap();
|
||||||
fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap();
|
fs::create_dir_all(store_path.join("raw").join("c").join("d")).unwrap();
|
||||||
fs::write(
|
fs::write(
|
||||||
|
|
|
||||||
184
crates/archivr-core/src/archive.rs
Normal file
184
crates/archivr-core/src/archive.rs
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use std::{
|
||||||
|
env, fs,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::database;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ArchivePaths {
|
||||||
|
pub archive_path: PathBuf,
|
||||||
|
pub store_path: PathBuf,
|
||||||
|
pub name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn find_archive_path_from(start: &Path) -> Result<Option<PathBuf>> {
|
||||||
|
let mut dir = start.to_path_buf();
|
||||||
|
loop {
|
||||||
|
let candidate = dir.join(".archivr");
|
||||||
|
if candidate.is_dir() {
|
||||||
|
return Ok(Some(candidate));
|
||||||
|
}
|
||||||
|
if !dir.pop() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn find_archive_path() -> Result<Option<PathBuf>> {
|
||||||
|
let cwd = env::current_dir().context("failed to read current working directory")?;
|
||||||
|
find_archive_path_from(&cwd)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read_archive_paths(archive_path: &Path) -> Result<ArchivePaths> {
|
||||||
|
if !archive_path.is_dir() {
|
||||||
|
bail!("archive path does not exist: {}", archive_path.display());
|
||||||
|
}
|
||||||
|
|
||||||
|
let name = fs::read_to_string(archive_path.join("name"))
|
||||||
|
.with_context(|| format!("failed to read archive name in {}", archive_path.display()))?
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
let store_path = fs::read_to_string(archive_path.join("store_path"))
|
||||||
|
.with_context(|| format!("failed to read store path in {}", archive_path.display()))?;
|
||||||
|
|
||||||
|
Ok(ArchivePaths {
|
||||||
|
archive_path: archive_path.to_path_buf(),
|
||||||
|
store_path: PathBuf::from(store_path.trim()),
|
||||||
|
name,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn initialize_archive(
|
||||||
|
archive_parent: &Path,
|
||||||
|
store_path: &Path,
|
||||||
|
archive_name: &str,
|
||||||
|
force_with_info_removal: bool,
|
||||||
|
) -> Result<ArchivePaths> {
|
||||||
|
let archive_path = archive_parent.join(".archivr");
|
||||||
|
|
||||||
|
if archive_path.exists() {
|
||||||
|
if !archive_path.is_dir() {
|
||||||
|
bail!(
|
||||||
|
"Archive path exists and is not a directory: {}",
|
||||||
|
archive_path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if force_with_info_removal {
|
||||||
|
fs::remove_dir_all(&archive_path)?;
|
||||||
|
} else if fs::read_dir(&archive_path)?.next().is_some() {
|
||||||
|
bail!(
|
||||||
|
"Archive already exists at {} and is not empty. Use --force-with-info-removal to reinitialize.",
|
||||||
|
archive_path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if store_path.exists() && !force_with_info_removal {
|
||||||
|
bail!("Store path already exists at {}", store_path.display());
|
||||||
|
}
|
||||||
|
|
||||||
|
fs::create_dir_all(&archive_path)?;
|
||||||
|
fs::create_dir_all(store_path)?;
|
||||||
|
fs::write(archive_path.join("name"), archive_name)?;
|
||||||
|
let canonical_store_path = store_path
|
||||||
|
.canonicalize()
|
||||||
|
.with_context(|| format!("failed to canonicalize {}", store_path.display()))?;
|
||||||
|
fs::write(
|
||||||
|
archive_path.join("store_path"),
|
||||||
|
canonical_store_path
|
||||||
|
.to_str()
|
||||||
|
.context("store path is not valid UTF-8")?,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
initialize_store_directories(&canonical_store_path)?;
|
||||||
|
let conn = database::open_or_initialize(&archive_path)?;
|
||||||
|
let _ = database::ensure_default_user(&conn)?;
|
||||||
|
|
||||||
|
Ok(ArchivePaths {
|
||||||
|
archive_path,
|
||||||
|
store_path: canonical_store_path,
|
||||||
|
name: archive_name.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn initialize_store_directories(store_path: &Path) -> Result<()> {
|
||||||
|
fs::create_dir_all(store_path.join("raw"))?;
|
||||||
|
fs::create_dir_all(store_path.join("raw_tweets"))?;
|
||||||
|
fs::create_dir_all(store_path.join("structured"))?;
|
||||||
|
fs::create_dir_all(store_path.join("temp"))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
fn unique_path(prefix: &str) -> PathBuf {
|
||||||
|
let nanos = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_nanos();
|
||||||
|
env::temp_dir().join(format!("{prefix}-{nanos}-{}", std::process::id()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn find_archive_path_walks_up_to_dot_archivr() {
|
||||||
|
let root = unique_path("archivr-core-find");
|
||||||
|
let nested = root.join("a").join("b");
|
||||||
|
fs::create_dir_all(root.join(".archivr")).unwrap();
|
||||||
|
fs::create_dir_all(&nested).unwrap();
|
||||||
|
|
||||||
|
let found = find_archive_path_from(&nested).unwrap().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(found, root.join(".archivr"));
|
||||||
|
let _ = fs::remove_dir_all(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_archive_paths_returns_name_and_store_path() {
|
||||||
|
let root = unique_path("archivr-core-open");
|
||||||
|
let archive_path = root.join(".archivr");
|
||||||
|
let store_path = root.join("store");
|
||||||
|
fs::create_dir_all(&archive_path).unwrap();
|
||||||
|
fs::create_dir_all(&store_path).unwrap();
|
||||||
|
fs::write(archive_path.join("name"), "Personal").unwrap();
|
||||||
|
fs::write(
|
||||||
|
archive_path.join("store_path"),
|
||||||
|
store_path.display().to_string(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let paths = read_archive_paths(&archive_path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(paths.archive_path, archive_path);
|
||||||
|
assert_eq!(paths.store_path, store_path);
|
||||||
|
assert_eq!(paths.name, "Personal");
|
||||||
|
let _ = fs::remove_dir_all(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn initialize_archive_creates_database_store_and_metadata() {
|
||||||
|
let root = unique_path("archivr-core-init");
|
||||||
|
let archive_parent = root.join("archive");
|
||||||
|
let store_path = root.join("store");
|
||||||
|
|
||||||
|
let paths = initialize_archive(&archive_parent, &store_path, "Personal", false).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(paths.archive_path, archive_parent.join(".archivr"));
|
||||||
|
assert!(
|
||||||
|
paths
|
||||||
|
.archive_path
|
||||||
|
.join(database::DATABASE_FILE_NAME)
|
||||||
|
.is_file()
|
||||||
|
);
|
||||||
|
assert!(paths.store_path.join("raw").is_dir());
|
||||||
|
assert!(paths.store_path.join("raw_tweets").is_dir());
|
||||||
|
assert!(paths.store_path.join("structured").is_dir());
|
||||||
|
assert!(paths.store_path.join("temp").is_dir());
|
||||||
|
let _ = fs::remove_dir_all(root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
pub mod archive;
|
||||||
pub mod database;
|
pub mod database;
|
||||||
pub mod downloader;
|
pub mod downloader;
|
||||||
pub mod hash;
|
pub mod hash;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue