mirror of
https://framagit.org/veretcle/scootaloo.git
synced 2025-07-20 17:11:19 +02:00
refactor: make everything a little more modular
This commit is contained in:
VC
51
src/config.rs
Normal file
51
src/config.rs
Normal file
@@ -0,0 +1,51 @@
|
||||
// std
|
||||
use std::fs::read_to_string;
|
||||
|
||||
// toml
|
||||
use serde::Deserialize;
|
||||
|
||||
/// General configuration Struct
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
pub twitter: TwitterConfig,
|
||||
pub mastodon: MastodonConfig,
|
||||
pub scootaloo: ScootalooConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TwitterConfig {
|
||||
pub username: String,
|
||||
pub consumer_key: String,
|
||||
pub consumer_secret: String,
|
||||
pub access_key: String,
|
||||
pub access_secret: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct MastodonConfig {
|
||||
pub base: String,
|
||||
pub client_id: String,
|
||||
pub client_secret: String,
|
||||
pub redirect: String,
|
||||
pub token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ScootalooConfig {
|
||||
pub db_path: String,
|
||||
pub cache_path: String,
|
||||
}
|
||||
|
||||
/// Parses the TOML file into a Config Struct
|
||||
pub fn parse_toml(toml_file: &str) -> Config {
|
||||
let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
|
||||
panic!("Cannot open config file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
|
||||
panic!("Cannot parse TOML file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
27
src/error.rs
Normal file
27
src/error.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ScootalooError {
|
||||
details: String,
|
||||
}
|
||||
|
||||
impl ScootalooError {
|
||||
pub fn new(msg: &str) -> ScootalooError {
|
||||
ScootalooError {
|
||||
details: String::from(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ScootalooError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "{}", self.details)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ScootalooError {
|
||||
fn description(&self) -> &str {
|
||||
&self.details
|
||||
}
|
||||
}
|
||||
|
||||
337
src/lib.rs
337
src/lib.rs
@@ -1,334 +1,53 @@
|
||||
// auto-imports
|
||||
mod error;
|
||||
use error::ScootalooError;
|
||||
|
||||
mod config;
|
||||
use config::Config;
|
||||
pub use config::parse_toml;
|
||||
|
||||
mod mastodon;
|
||||
use mastodon::{get_mastodon_token, build_basic_status};
|
||||
pub use mastodon::register;
|
||||
|
||||
mod twitter;
|
||||
use twitter::*;
|
||||
|
||||
mod util;
|
||||
|
||||
mod state;
|
||||
use state::{read_state, write_state};
|
||||
pub use state::init_db;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::HashMap,
|
||||
io::stdin,
|
||||
fmt,
|
||||
fs::{read_to_string, write},
|
||||
error::Error,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
|
||||
// toml
|
||||
use serde::Deserialize;
|
||||
|
||||
// egg-mode
|
||||
use egg_mode::{
|
||||
Token,
|
||||
KeyPair,
|
||||
entities::{UrlEntity, MediaEntity, MentionEntity, MediaType},
|
||||
user::UserID,
|
||||
tweet::{
|
||||
Tweet,
|
||||
user_timeline,
|
||||
},
|
||||
};
|
||||
// tokio
|
||||
use tokio::fs::remove_file;
|
||||
|
||||
// elefren
|
||||
use elefren::{
|
||||
prelude::*,
|
||||
apps::App,
|
||||
status_builder::StatusBuilder,
|
||||
scopes::Scopes,
|
||||
};
|
||||
|
||||
// reqwest
|
||||
use reqwest::Url;
|
||||
|
||||
// tokio
|
||||
use tokio::{
|
||||
io::copy,
|
||||
fs::{File, create_dir_all, remove_file},
|
||||
};
|
||||
|
||||
// htmlescape
|
||||
use htmlescape::decode_html;
|
||||
|
||||
// log
|
||||
use log::{info, warn, error, debug};
|
||||
|
||||
/**********
|
||||
* Generic usage functions
|
||||
***********/
|
||||
/*
|
||||
* Those functions are related to the Twitter side of things
|
||||
*/
|
||||
/// Reads last tweet id from a file
|
||||
fn read_state(s: &str) -> Option<u64> {
|
||||
let state = read_to_string(s);
|
||||
|
||||
if let Ok(s) = state {
|
||||
debug!("Last Tweet ID (from file): {}", &s);
|
||||
return s.parse::<u64>().ok();
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Writes last treated tweet id to a file
|
||||
fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
|
||||
write(f, format!("{}", s))
|
||||
}
|
||||
|
||||
/// Gets Twitter oauth2 token
|
||||
fn get_oauth2_token(config: &Config) -> Token {
|
||||
let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
|
||||
let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
|
||||
|
||||
Token::Access {
|
||||
consumer: con_token,
|
||||
access: access_token,
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets Twitter user timeline
|
||||
async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
|
||||
// fix the page size to 200 as it is the maximum Twitter authorizes
|
||||
let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
|
||||
.with_page_size(200)
|
||||
.older(lid)
|
||||
.await?;
|
||||
|
||||
Ok(feed.to_vec())
|
||||
}
|
||||
|
||||
/// Decodes urls from UrlEntities
|
||||
fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_urls = HashMap::new();
|
||||
|
||||
for url in urls {
|
||||
if url.expanded_url.is_some() {
|
||||
// unwrap is safe here as we just verified that there is something inside expanded_url
|
||||
decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
decoded_urls
|
||||
}
|
||||
|
||||
/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
|
||||
/// Fediverse
|
||||
fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_mentions = HashMap::new();
|
||||
|
||||
for um in ums {
|
||||
decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
|
||||
}
|
||||
|
||||
decoded_mentions
|
||||
}
|
||||
|
||||
/// Retrieves a single media from a tweet and store it in a temporary file
|
||||
async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
match m.media_type {
|
||||
MediaType::Photo => {
|
||||
return cache_media(&m.media_url_https, t).await;
|
||||
},
|
||||
_ => {
|
||||
match &m.video_info {
|
||||
Some(v) => {
|
||||
for variant in &v.variants {
|
||||
if variant.content_type == "video/mp4" {
|
||||
return cache_media(&variant.url, t).await;
|
||||
}
|
||||
}
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
|
||||
},
|
||||
None => {
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* Those functions are related to the Mastodon side of things
|
||||
*/
|
||||
/// Gets Mastodon Data
|
||||
fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
|
||||
let data = Data {
|
||||
base: Cow::from(String::from(&masto.base)),
|
||||
client_id: Cow::from(String::from(&masto.client_id)),
|
||||
client_secret: Cow::from(String::from(&masto.client_secret)),
|
||||
redirect: Cow::from(String::from(&masto.redirect)),
|
||||
token: Cow::from(String::from(&masto.token)),
|
||||
};
|
||||
|
||||
Mastodon::from(data)
|
||||
}
|
||||
|
||||
/// Builds toot text from tweet
|
||||
fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
|
||||
let mut toot = String::from(&tweet.text);
|
||||
|
||||
let decoded_urls = decode_urls(&tweet.entities.urls);
|
||||
|
||||
for decoded_url in decoded_urls {
|
||||
toot = toot.replace(&decoded_url.0, &decoded_url.1);
|
||||
}
|
||||
|
||||
let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
|
||||
|
||||
for decoded_mention in decoded_mentions {
|
||||
toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
|
||||
}
|
||||
|
||||
if let Ok(t) = decode_html(&toot) {
|
||||
toot = t;
|
||||
}
|
||||
|
||||
Ok(toot)
|
||||
}
|
||||
|
||||
/*
|
||||
* Generic private functions
|
||||
*/
|
||||
/// Gets and caches Twitter Media inside the determined temp dir
|
||||
async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
// create dir
|
||||
create_dir_all(t).await?;
|
||||
|
||||
// get file
|
||||
let mut response = reqwest::get(u).await?;
|
||||
|
||||
// create local file
|
||||
let url = Url::parse(u)?;
|
||||
let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
|
||||
.last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
|
||||
|
||||
let dest_filepath = format!("{}/{}", t, dest_filename);
|
||||
|
||||
let mut dest_file = File::create(&dest_filepath).await?;
|
||||
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
copy(&mut &*chunk, &mut dest_file).await?;
|
||||
}
|
||||
|
||||
Ok(dest_filepath)
|
||||
}
|
||||
|
||||
/**********
|
||||
* local error handler
|
||||
**********/
|
||||
#[derive(Debug)]
|
||||
struct ScootalooError {
|
||||
details: String,
|
||||
}
|
||||
|
||||
impl ScootalooError {
|
||||
fn new(msg: &str) -> ScootalooError {
|
||||
ScootalooError {
|
||||
details: String::from(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ScootalooError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "{}", self.details)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ScootalooError {
|
||||
fn description(&self) -> &str {
|
||||
&self.details
|
||||
}
|
||||
}
|
||||
|
||||
/**********
|
||||
* Config structure
|
||||
***********/
|
||||
/// General configuration Struct
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
twitter: TwitterConfig,
|
||||
mastodon: MastodonConfig,
|
||||
scootaloo: ScootalooConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TwitterConfig {
|
||||
username: String,
|
||||
consumer_key: String,
|
||||
consumer_secret: String,
|
||||
access_key: String,
|
||||
access_secret: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct MastodonConfig {
|
||||
base: String,
|
||||
client_id: String,
|
||||
client_secret: String,
|
||||
redirect: String,
|
||||
token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ScootalooConfig {
|
||||
last_tweet_path: String,
|
||||
cache_path: String,
|
||||
}
|
||||
|
||||
/*********
|
||||
* Main functions
|
||||
*********/
|
||||
/// Parses the TOML file into a Config Struct
|
||||
pub fn parse_toml(toml_file: &str) -> Config {
|
||||
let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
|
||||
panic!("Cannot open config file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
|
||||
panic!("Cannot parse TOML file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
/// Generic register function
|
||||
/// As this function is supposed to be run only once, it will panic for every error it encounters
|
||||
/// Most of this function is a direct copy/paste of the official `elefren` crate
|
||||
pub fn register(host: &str) {
|
||||
let mut builder = App::builder();
|
||||
builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
|
||||
.redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
|
||||
.scopes(Scopes::write_all())
|
||||
.website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
|
||||
|
||||
let app = builder.build().expect("Cannot build the app");
|
||||
|
||||
let registration = Registration::new(host).register(app).expect("Cannot build registration object");
|
||||
let url = registration.authorize_url().expect("Cannot generate registration URI!");
|
||||
|
||||
println!("Click this link to authorize on Mastodon: {}", url);
|
||||
println!("Paste the returned authorization code: ");
|
||||
|
||||
let mut input = String::new();
|
||||
stdin().read_line(&mut input).expect("Unable to read back registration code!");
|
||||
|
||||
let code = input.trim();
|
||||
let mastodon = registration.complete(code).expect("Unable to create access token!");
|
||||
|
||||
let toml = toml::to_string(&*mastodon).unwrap();
|
||||
|
||||
println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
|
||||
}
|
||||
|
||||
/// This is where the magic happens
|
||||
#[tokio::main]
|
||||
pub async fn run(config: Config) {
|
||||
// retrieve the last tweet ID for the username
|
||||
let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
|
||||
let last_tweet_id = read_state(&config.scootaloo.db_path);
|
||||
|
||||
// get OAuth2 token
|
||||
let token = get_oauth2_token(&config);
|
||||
let token = get_oauth2_token(&config.twitter);
|
||||
|
||||
// get Mastodon instance
|
||||
let mastodon = get_mastodon_token(&config.mastodon);
|
||||
|
||||
// get user timeline feed (Vec<tweet>)
|
||||
let mut feed = get_user_timeline(&config, token, last_tweet_id)
|
||||
let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
|
||||
.await
|
||||
.unwrap_or_else(|e|
|
||||
panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
|
||||
@@ -411,7 +130,7 @@ pub async fn run(config: Config) {
|
||||
// last_tweet gathered not to be written
|
||||
|
||||
// write the current state (tweet ID) to avoid copying it another time
|
||||
write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e|
|
||||
write_state(&config.scootaloo.db_path, tweet.id).unwrap_or_else(|e|
|
||||
panic!("Can’t write the last tweet retrieved: {}", e)
|
||||
);
|
||||
}
|
||||
|
||||
29
src/main.rs
29
src/main.rs
@@ -11,6 +11,8 @@ use simple_logger::SimpleLogger;
|
||||
// std
|
||||
use std::str::FromStr;
|
||||
|
||||
const DEFAULT_CONFIG_PATH: &'static str = "/usr/local/etc/scootaloo.toml";
|
||||
|
||||
fn main() {
|
||||
let matches = App::new(env!("CARGO_PKG_NAME"))
|
||||
.version(env!("CARGO_PKG_VERSION"))
|
||||
@@ -40,10 +42,29 @@ fn main() {
|
||||
.takes_value(true)
|
||||
.required(true)
|
||||
.display_order(1)))
|
||||
.subcommand(SubCommand::with_name("init")
|
||||
.version(env!("CARGO_PKG_VERSION"))
|
||||
.about("Command to init Scootaloo DB")
|
||||
.arg(Arg::with_name("config")
|
||||
.short("c")
|
||||
.long("config")
|
||||
.value_name("CONFIG_FILE")
|
||||
.help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml")
|
||||
.takes_value(true)
|
||||
.display_order(1)))
|
||||
.get_matches();
|
||||
if let Some(matches) = matches.subcommand_matches("register") {
|
||||
register(matches.value_of("host").unwrap());
|
||||
return;
|
||||
|
||||
match matches.subcommand() {
|
||||
("register", Some(sub_m)) => {
|
||||
register(sub_m.value_of("host").unwrap());
|
||||
return;
|
||||
},
|
||||
("init", Some(sub_m)) => {
|
||||
let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
|
||||
init_db(&config).unwrap();
|
||||
return;
|
||||
},
|
||||
_ => (),
|
||||
}
|
||||
|
||||
if matches.is_present("log_level") {
|
||||
@@ -56,7 +77,7 @@ fn main() {
|
||||
};
|
||||
}
|
||||
|
||||
let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
|
||||
let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
|
||||
|
||||
run(config);
|
||||
}
|
||||
|
||||
106
src/mastodon.rs
Normal file
106
src/mastodon.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
// auto imports
|
||||
use crate::config::MastodonConfig;
|
||||
use crate::twitter::decode_urls;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
error::Error,
|
||||
collections::HashMap,
|
||||
io::stdin,
|
||||
};
|
||||
|
||||
// htmlescape
|
||||
use htmlescape::decode_html;
|
||||
|
||||
// egg-mode
|
||||
use egg_mode::{
|
||||
tweet::Tweet,
|
||||
entities::MentionEntity,
|
||||
};
|
||||
|
||||
// elefren
|
||||
use elefren::{
|
||||
prelude::*,
|
||||
apps::App,
|
||||
scopes::Scopes,
|
||||
};
|
||||
|
||||
|
||||
/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
|
||||
/// Fediverse
|
||||
fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_mentions = HashMap::new();
|
||||
|
||||
for um in ums {
|
||||
decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
|
||||
}
|
||||
|
||||
decoded_mentions
|
||||
}
|
||||
|
||||
/// Gets Mastodon Data
|
||||
pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
|
||||
let data = Data {
|
||||
base: Cow::from(String::from(&masto.base)),
|
||||
client_id: Cow::from(String::from(&masto.client_id)),
|
||||
client_secret: Cow::from(String::from(&masto.client_secret)),
|
||||
redirect: Cow::from(String::from(&masto.redirect)),
|
||||
token: Cow::from(String::from(&masto.token)),
|
||||
};
|
||||
|
||||
Mastodon::from(data)
|
||||
}
|
||||
|
||||
/// Builds toot text from tweet
|
||||
pub fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
|
||||
let mut toot = String::from(&tweet.text);
|
||||
|
||||
let decoded_urls = decode_urls(&tweet.entities.urls);
|
||||
|
||||
for decoded_url in decoded_urls {
|
||||
toot = toot.replace(&decoded_url.0, &decoded_url.1);
|
||||
}
|
||||
|
||||
let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
|
||||
|
||||
for decoded_mention in decoded_mentions {
|
||||
toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
|
||||
}
|
||||
|
||||
if let Ok(t) = decode_html(&toot) {
|
||||
toot = t;
|
||||
}
|
||||
|
||||
Ok(toot)
|
||||
}
|
||||
|
||||
/// Generic register function
|
||||
/// As this function is supposed to be run only once, it will panic for every error it encounters
|
||||
/// Most of this function is a direct copy/paste of the official `elefren` crate
|
||||
pub fn register(host: &str) {
|
||||
let mut builder = App::builder();
|
||||
builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
|
||||
.redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
|
||||
.scopes(Scopes::write_all())
|
||||
.website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
|
||||
|
||||
let app = builder.build().expect("Cannot build the app");
|
||||
|
||||
let registration = Registration::new(host).register(app).expect("Cannot build registration object");
|
||||
let url = registration.authorize_url().expect("Cannot generate registration URI!");
|
||||
|
||||
println!("Click this link to authorize on Mastodon: {}", url);
|
||||
println!("Paste the returned authorization code: ");
|
||||
|
||||
let mut input = String::new();
|
||||
stdin().read_line(&mut input).expect("Unable to read back registration code!");
|
||||
|
||||
let code = input.trim();
|
||||
let mastodon = registration.complete(code).expect("Unable to create access token!");
|
||||
|
||||
let toml = toml::to_string(&*mastodon).unwrap();
|
||||
|
||||
println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
|
||||
}
|
||||
|
||||
38
src/state.rs
Normal file
38
src/state.rs
Normal file
@@ -0,0 +1,38 @@
|
||||
// auto-imports
|
||||
use crate::config::Config;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
fs::{read_to_string, write},
|
||||
error::Error,
|
||||
};
|
||||
|
||||
// log
|
||||
use log::debug;
|
||||
|
||||
/// Reads last tweet id from a file
|
||||
pub fn read_state(s: &str) -> Option<u64> {
|
||||
let state = read_to_string(s);
|
||||
|
||||
if let Ok(s) = state {
|
||||
debug!("Last Tweet ID (from file): {}", &s);
|
||||
return s.parse::<u64>().ok();
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Writes last treated tweet id to a file
|
||||
pub fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
|
||||
write(f, format!("{}", s))
|
||||
}
|
||||
|
||||
/*********
|
||||
* Main functions
|
||||
*********/
|
||||
/// Initiates the DB from path
|
||||
pub fn init_db(config: &Config) -> Result<(), Box<dyn Error>> {
|
||||
println!("config.scootaloo.db_path: {}", config.scootaloo.db_path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
83
src/twitter.rs
Normal file
83
src/twitter.rs
Normal file
@@ -0,0 +1,83 @@
|
||||
// auto-imports
|
||||
use crate::ScootalooError;
|
||||
use crate::config::TwitterConfig;
|
||||
use crate::util::cache_media;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
error::Error,
|
||||
collections::HashMap,
|
||||
};
|
||||
|
||||
// egg-mode
|
||||
use egg_mode::{
|
||||
Token,
|
||||
KeyPair,
|
||||
entities::{UrlEntity, MediaEntity, MediaType},
|
||||
user::UserID,
|
||||
tweet::{
|
||||
Tweet,
|
||||
user_timeline,
|
||||
},
|
||||
};
|
||||
|
||||
/// Gets Twitter oauth2 token
|
||||
pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
|
||||
let con_token = KeyPair::new(String::from(&config.consumer_key), String::from(&config.consumer_secret));
|
||||
let access_token = KeyPair::new(String::from(&config.access_key), String::from(&config.access_secret));
|
||||
|
||||
Token::Access {
|
||||
consumer: con_token,
|
||||
access: access_token,
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets Twitter user timeline
|
||||
pub async fn get_user_timeline(config: &TwitterConfig, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
|
||||
// fix the page size to 200 as it is the maximum Twitter authorizes
|
||||
let (_, feed) = user_timeline(UserID::from(String::from(&config.username)), true, false, &token)
|
||||
.with_page_size(200)
|
||||
.older(lid)
|
||||
.await?;
|
||||
|
||||
Ok(feed.to_vec())
|
||||
}
|
||||
|
||||
/// Decodes urls from UrlEntities
|
||||
pub fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_urls = HashMap::new();
|
||||
|
||||
for url in urls {
|
||||
if url.expanded_url.is_some() {
|
||||
// unwrap is safe here as we just verified that there is something inside expanded_url
|
||||
decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
decoded_urls
|
||||
}
|
||||
|
||||
/// Retrieves a single media from a tweet and store it in a temporary file
|
||||
pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
match m.media_type {
|
||||
MediaType::Photo => {
|
||||
return cache_media(&m.media_url_https, t).await;
|
||||
},
|
||||
_ => {
|
||||
match &m.video_info {
|
||||
Some(v) => {
|
||||
for variant in &v.variants {
|
||||
if variant.content_type == "video/mp4" {
|
||||
return cache_media(&variant.url, t).await;
|
||||
}
|
||||
}
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
|
||||
},
|
||||
None => {
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
37
src/util.rs
Normal file
37
src/util.rs
Normal file
@@ -0,0 +1,37 @@
|
||||
// std
|
||||
use std::error::Error;
|
||||
use crate::ScootalooError;
|
||||
|
||||
// reqwest
|
||||
use reqwest::Url;
|
||||
|
||||
// tokio
|
||||
use tokio::{
|
||||
io::copy,
|
||||
fs::{File, create_dir_all},
|
||||
};
|
||||
|
||||
/// Gets and caches Twitter Media inside the determined temp dir
|
||||
pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
// create dir
|
||||
create_dir_all(t).await?;
|
||||
|
||||
// get file
|
||||
let mut response = reqwest::get(u).await?;
|
||||
|
||||
// create local file
|
||||
let url = Url::parse(u)?;
|
||||
let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
|
||||
.last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
|
||||
|
||||
let dest_filepath = format!("{}/{}", t, dest_filename);
|
||||
|
||||
let mut dest_file = File::create(&dest_filepath).await?;
|
||||
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
copy(&mut &*chunk, &mut dest_file).await?;
|
||||
}
|
||||
|
||||
Ok(dest_filepath)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user