fix typos in .gitignore
This commit is contained in:
parent
35057d7957
commit
6eb3097f3d
6 changed files with 241 additions and 2 deletions
1
src/scrapers/mod.rs
Normal file
1
src/scrapers/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod twitter;
|
||||
24
src/scrapers/twitter.rs
Normal file
24
src/scrapers/twitter.rs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
use anyhow::{Context, Result, bail};
|
||||
use std::{path::PathBuf, process::Command};
|
||||
|
||||
pub fn scrape(url: &str) -> Result<PathBuf> {
|
||||
let tweet_id = url.split('/').next_back().unwrap();
|
||||
println!("Scraping tweet ID: {}", tweet_id);
|
||||
|
||||
let out = Command::new("python")
|
||||
.arg("scrape_user_tweet_contents.py")
|
||||
.arg("--tweet-ids")
|
||||
.arg(tweet_id)
|
||||
.output()
|
||||
.with_context(|| "Failed to execute tweet scraping command")?;
|
||||
println!("Output command: {:?}", out);
|
||||
|
||||
if PathBuf::from("scraped-tweets")
|
||||
.join(format!("tweet-{}.toml", tweet_id))
|
||||
.exists()
|
||||
{
|
||||
return Ok(PathBuf::from("scraped-tweets").join(format!("tweet-{}.toml", tweet_id)));
|
||||
}
|
||||
|
||||
bail!("Scraping failed for tweet: {}", url)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue