diff --git a/Cargo.lock b/Cargo.lock index 53f7909..3924ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.6", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.15" @@ -550,6 +561,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "flate2" version = "1.0.20" @@ -755,9 +778,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", @@ -819,6 +842,24 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -1027,7 +1068,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1123,6 +1164,16 @@ version = "0.2.124" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +[[package]] +name = "libsqlite3-sys" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.3.4" @@ -1340,9 +1391,9 @@ checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" [[package]] name = "once_cell" -version = "1.7.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "opaque-debug" @@ -1698,7 +1749,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.6", ] [[package]] @@ -1899,6 +1950,21 @@ dependencies = [ "winreg 0.7.0", ] +[[package]] +name = "rusqlite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "memchr", + "smallvec 1.6.1", +] + [[package]] name = "rustc-demangle" version = "0.1.18" @@ -1947,7 +2013,7 @@ dependencies = [ [[package]] name = "scootaloo" -version = "0.4.2" +version = "0.5.0" dependencies = [ "clap", "egg-mode", @@ -1956,6 +2022,7 @@ dependencies = [ "htmlescape", "log", "reqwest 0.11.3", + "rusqlite", "serde", "simple_logger", "tokio 1.5.0", diff --git a/Cargo.toml b/Cargo.toml index 724e30f..2aefd55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scootaloo" -version = "0.4.2" +version = "0.5.0" authors = ["VC "] edition = "2021" @@ -12,6 +12,7 @@ toml = "^0.5" clap = "^2.34" futures = "^0.3" egg-mode = "^0.16" +rusqlite = "^0.27" tokio = { version = "1", features = ["full"]} elefren = "^0.22" htmlescape = "^0.3" diff --git a/README.md b/README.md index 538ac71..f6f53ed 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ It: If any of the last steps failed, the Toot gets published with the exact same text as the Tweet. -RT are excluded, replies are included.but only the source threads are copied, not the actual replies to other Twitter users. +RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users. # Usage @@ -16,7 +16,7 @@ First up, create a configuration file (default path is `/usr/local/etc/scootaloo ```toml [scootaloo] -last_tweet_path="/usr/local/etc/last_tweet" ## file containing the last tweet id received, must be writable +db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable [twitter] @@ -29,6 +29,11 @@ access_key="MYACCESSKEY" access_secret="MYACCESSSECRET" ``` +Then run the command with the `init` subcommand to initiate the DB: +``` +scootaloo init +``` + Then run the command with the `register` subcommand: ```sh scootaloo register --host https://m.nintendojo.fr diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..d0e2881 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,51 @@ +// std +use std::fs::read_to_string; + +// toml +use serde::Deserialize; + +/// General configuration Struct +#[derive(Debug, Deserialize)] +pub struct Config { + pub twitter: TwitterConfig, + pub mastodon: MastodonConfig, + pub scootaloo: ScootalooConfig, +} + +#[derive(Debug, Deserialize)] +pub struct TwitterConfig { + pub username: String, + pub consumer_key: String, + pub consumer_secret: String, + pub access_key: String, + pub access_secret: String, +} + +#[derive(Debug, Deserialize)] +pub struct MastodonConfig { + pub base: String, + pub client_id: String, + pub client_secret: String, + pub redirect: String, + pub token: String, +} + +#[derive(Debug, Deserialize)] +pub struct ScootalooConfig { + pub db_path: String, + pub cache_path: String, +} + +/// Parses the TOML file into a Config Struct +pub fn parse_toml(toml_file: &str) -> Config { + let toml_config = read_to_string(toml_file).unwrap_or_else(|e| + panic!("Cannot open config file {}: {}", toml_file, e) + ); + + let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e| + panic!("Cannot parse TOML file {}: {}", toml_file, e) + ); + + config +} + diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..015294c --- /dev/null +++ b/src/error.rs @@ -0,0 +1,27 @@ +use std::fmt; + +#[derive(Debug)] +pub struct ScootalooError { + details: String, +} + +impl ScootalooError { + pub fn new(msg: &str) -> ScootalooError { + ScootalooError { + details: String::from(msg), + } + } +} + +impl fmt::Display for ScootalooError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.details) + } +} + +impl std::error::Error for ScootalooError { + fn description(&self) -> &str { + &self.details + } +} + diff --git a/src/lib.rs b/src/lib.rs index 810e944..c2b2e9d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,334 +1,53 @@ +// auto-imports +mod error; +use error::ScootalooError; + +mod config; +use config::Config; +pub use config::parse_toml; + +mod mastodon; +use mastodon::{get_mastodon_token, build_basic_status}; +pub use mastodon::register; + +mod twitter; +use twitter::*; + +mod util; + +mod state; +use state::{read_state, write_state}; +pub use state::init_db; + // std -use std::{ - borrow::Cow, - collections::HashMap, - io::stdin, - fmt, - fs::{read_to_string, write}, - error::Error, -}; +use std::borrow::Cow; -// toml -use serde::Deserialize; - -// egg-mode -use egg_mode::{ - Token, - KeyPair, - entities::{UrlEntity, MediaEntity, MentionEntity, MediaType}, - user::UserID, - tweet::{ - Tweet, - user_timeline, - }, -}; +// tokio +use tokio::fs::remove_file; // elefren use elefren::{ prelude::*, - apps::App, status_builder::StatusBuilder, - scopes::Scopes, }; -// reqwest -use reqwest::Url; - -// tokio -use tokio::{ - io::copy, - fs::{File, create_dir_all, remove_file}, -}; - -// htmlescape -use htmlescape::decode_html; - // log use log::{info, warn, error, debug}; -/********** - * Generic usage functions -***********/ -/* - * Those functions are related to the Twitter side of things - */ -/// Reads last tweet id from a file -fn read_state(s: &str) -> Option { - let state = read_to_string(s); - - if let Ok(s) = state { - debug!("Last Tweet ID (from file): {}", &s); - return s.parse::().ok(); - } - - None -} - -/// Writes last treated tweet id to a file -fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> { - write(f, format!("{}", s)) -} - -/// Gets Twitter oauth2 token -fn get_oauth2_token(config: &Config) -> Token { - let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret)); - let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret)); - - Token::Access { - consumer: con_token, - access: access_token, - } -} - -/// Gets Twitter user timeline -async fn get_user_timeline(config: &Config, token: Token, lid: Option) -> Result, Box> { - // fix the page size to 200 as it is the maximum Twitter authorizes - let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token) - .with_page_size(200) - .older(lid) - .await?; - - Ok(feed.to_vec()) -} - -/// Decodes urls from UrlEntities -fn decode_urls(urls: &Vec) -> HashMap { - let mut decoded_urls = HashMap::new(); - - for url in urls { - if url.expanded_url.is_some() { - // unwrap is safe here as we just verified that there is something inside expanded_url - decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap())); - } - } - - decoded_urls -} - -/// Decodes the Twitter mention to something that will make sense once Twitter has joined the -/// Fediverse -fn twitter_mentions(ums: &Vec) -> HashMap { - let mut decoded_mentions = HashMap::new(); - - for um in ums { - decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name)); - } - - decoded_mentions -} - -/// Retrieves a single media from a tweet and store it in a temporary file -async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result> { - match m.media_type { - MediaType::Photo => { - return cache_media(&m.media_url_https, t).await; - }, - _ => { - match &m.video_info { - Some(v) => { - for variant in &v.variants { - if variant.content_type == "video/mp4" { - return cache_media(&variant.url, t).await; - } - } - return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into()); - }, - None => { - return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into()); - }, - } - }, - }; -} - -/* - * Those functions are related to the Mastodon side of things - */ -/// Gets Mastodon Data -fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon { - let data = Data { - base: Cow::from(String::from(&masto.base)), - client_id: Cow::from(String::from(&masto.client_id)), - client_secret: Cow::from(String::from(&masto.client_secret)), - redirect: Cow::from(String::from(&masto.redirect)), - token: Cow::from(String::from(&masto.token)), - }; - - Mastodon::from(data) -} - -/// Builds toot text from tweet -fn build_basic_status(tweet: &Tweet) -> Result> { - let mut toot = String::from(&tweet.text); - - let decoded_urls = decode_urls(&tweet.entities.urls); - - for decoded_url in decoded_urls { - toot = toot.replace(&decoded_url.0, &decoded_url.1); - } - - let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions); - - for decoded_mention in decoded_mentions { - toot = toot.replace(&decoded_mention.0, &decoded_mention.1); - } - - if let Ok(t) = decode_html(&toot) { - toot = t; - } - - Ok(toot) -} - -/* - * Generic private functions - */ -/// Gets and caches Twitter Media inside the determined temp dir -async fn cache_media(u: &str, t: &str) -> Result> { - // create dir - create_dir_all(t).await?; - - // get file - let mut response = reqwest::get(u).await?; - - // create local file - let url = Url::parse(u)?; - let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))? - .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?; - - let dest_filepath = format!("{}/{}", t, dest_filename); - - let mut dest_file = File::create(&dest_filepath).await?; - - while let Some(chunk) = response.chunk().await? { - copy(&mut &*chunk, &mut dest_file).await?; - } - - Ok(dest_filepath) -} - -/********** - * local error handler -**********/ -#[derive(Debug)] -struct ScootalooError { - details: String, -} - -impl ScootalooError { - fn new(msg: &str) -> ScootalooError { - ScootalooError { - details: String::from(msg), - } - } -} - -impl fmt::Display for ScootalooError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.details) - } -} - -impl std::error::Error for ScootalooError { - fn description(&self) -> &str { - &self.details - } -} - -/********** - * Config structure -***********/ -/// General configuration Struct -#[derive(Debug, Deserialize)] -pub struct Config { - twitter: TwitterConfig, - mastodon: MastodonConfig, - scootaloo: ScootalooConfig, -} - -#[derive(Debug, Deserialize)] -struct TwitterConfig { - username: String, - consumer_key: String, - consumer_secret: String, - access_key: String, - access_secret: String, -} - -#[derive(Debug, Deserialize)] -struct MastodonConfig { - base: String, - client_id: String, - client_secret: String, - redirect: String, - token: String, -} - -#[derive(Debug, Deserialize)] -struct ScootalooConfig { - last_tweet_path: String, - cache_path: String, -} - -/********* - * Main functions -*********/ -/// Parses the TOML file into a Config Struct -pub fn parse_toml(toml_file: &str) -> Config { - let toml_config = read_to_string(toml_file).unwrap_or_else(|e| - panic!("Cannot open config file {}: {}", toml_file, e) - ); - - let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e| - panic!("Cannot parse TOML file {}: {}", toml_file, e) - ); - - config -} - -/// Generic register function -/// As this function is supposed to be run only once, it will panic for every error it encounters -/// Most of this function is a direct copy/paste of the official `elefren` crate -pub fn register(host: &str) { - let mut builder = App::builder(); - builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME")))) - .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob"))) - .scopes(Scopes::write_all()) - .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo"))); - - let app = builder.build().expect("Cannot build the app"); - - let registration = Registration::new(host).register(app).expect("Cannot build registration object"); - let url = registration.authorize_url().expect("Cannot generate registration URI!"); - - println!("Click this link to authorize on Mastodon: {}", url); - println!("Paste the returned authorization code: "); - - let mut input = String::new(); - stdin().read_line(&mut input).expect("Unable to read back registration code!"); - - let code = input.trim(); - let mastodon = registration.complete(code).expect("Unable to create access token!"); - - let toml = toml::to_string(&*mastodon).unwrap(); - - println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml); -} - /// This is where the magic happens #[tokio::main] pub async fn run(config: Config) { // retrieve the last tweet ID for the username - let last_tweet_id = read_state(&config.scootaloo.last_tweet_path); + let last_tweet_id = read_state(&config.scootaloo.db_path); // get OAuth2 token - let token = get_oauth2_token(&config); + let token = get_oauth2_token(&config.twitter); // get Mastodon instance let mastodon = get_mastodon_token(&config.mastodon); // get user timeline feed (Vec) - let mut feed = get_user_timeline(&config, token, last_tweet_id) + let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id) .await .unwrap_or_else(|e| panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e) @@ -411,7 +130,7 @@ pub async fn run(config: Config) { // last_tweet gathered not to be written // write the current state (tweet ID) to avoid copying it another time - write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e| + write_state(&config.scootaloo.db_path, tweet.id).unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e) ); } diff --git a/src/main.rs b/src/main.rs index 529faa7..77bb07d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,8 @@ use simple_logger::SimpleLogger; // std use std::str::FromStr; +const DEFAULT_CONFIG_PATH: &'static str = "/usr/local/etc/scootaloo.toml"; + fn main() { let matches = App::new(env!("CARGO_PKG_NAME")) .version(env!("CARGO_PKG_VERSION")) @@ -40,10 +42,29 @@ fn main() { .takes_value(true) .required(true) .display_order(1))) + .subcommand(SubCommand::with_name("init") + .version(env!("CARGO_PKG_VERSION")) + .about("Command to init Scootaloo DB") + .arg(Arg::with_name("config") + .short("c") + .long("config") + .value_name("CONFIG_FILE") + .help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml") + .takes_value(true) + .display_order(1))) .get_matches(); - if let Some(matches) = matches.subcommand_matches("register") { - register(matches.value_of("host").unwrap()); - return; + + match matches.subcommand() { + ("register", Some(sub_m)) => { + register(sub_m.value_of("host").unwrap()); + return; + }, + ("init", Some(sub_m)) => { + let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH)); + init_db(&config).unwrap(); + return; + }, + _ => (), } if matches.is_present("log_level") { @@ -56,7 +77,7 @@ fn main() { }; } - let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml")); + let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH)); run(config); } diff --git a/src/mastodon.rs b/src/mastodon.rs new file mode 100644 index 0000000..3f93e9d --- /dev/null +++ b/src/mastodon.rs @@ -0,0 +1,106 @@ +// auto imports +use crate::config::MastodonConfig; +use crate::twitter::decode_urls; + +// std +use std::{ + borrow::Cow, + error::Error, + collections::HashMap, + io::stdin, +}; + +// htmlescape +use htmlescape::decode_html; + +// egg-mode +use egg_mode::{ + tweet::Tweet, + entities::MentionEntity, +}; + +// elefren +use elefren::{ + prelude::*, + apps::App, + scopes::Scopes, +}; + + +/// Decodes the Twitter mention to something that will make sense once Twitter has joined the +/// Fediverse +fn twitter_mentions(ums: &Vec) -> HashMap { + let mut decoded_mentions = HashMap::new(); + + for um in ums { + decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name)); + } + + decoded_mentions +} + +/// Gets Mastodon Data +pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon { + let data = Data { + base: Cow::from(String::from(&masto.base)), + client_id: Cow::from(String::from(&masto.client_id)), + client_secret: Cow::from(String::from(&masto.client_secret)), + redirect: Cow::from(String::from(&masto.redirect)), + token: Cow::from(String::from(&masto.token)), + }; + + Mastodon::from(data) +} + +/// Builds toot text from tweet +pub fn build_basic_status(tweet: &Tweet) -> Result> { + let mut toot = String::from(&tweet.text); + + let decoded_urls = decode_urls(&tweet.entities.urls); + + for decoded_url in decoded_urls { + toot = toot.replace(&decoded_url.0, &decoded_url.1); + } + + let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions); + + for decoded_mention in decoded_mentions { + toot = toot.replace(&decoded_mention.0, &decoded_mention.1); + } + + if let Ok(t) = decode_html(&toot) { + toot = t; + } + + Ok(toot) +} + +/// Generic register function +/// As this function is supposed to be run only once, it will panic for every error it encounters +/// Most of this function is a direct copy/paste of the official `elefren` crate +pub fn register(host: &str) { + let mut builder = App::builder(); + builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME")))) + .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob"))) + .scopes(Scopes::write_all()) + .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo"))); + + let app = builder.build().expect("Cannot build the app"); + + let registration = Registration::new(host).register(app).expect("Cannot build registration object"); + let url = registration.authorize_url().expect("Cannot generate registration URI!"); + + println!("Click this link to authorize on Mastodon: {}", url); + println!("Paste the returned authorization code: "); + + let mut input = String::new(); + stdin().read_line(&mut input).expect("Unable to read back registration code!"); + + let code = input.trim(); + let mastodon = registration.complete(code).expect("Unable to create access token!"); + + let toml = toml::to_string(&*mastodon).unwrap(); + + println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml); +} + diff --git a/src/state.rs b/src/state.rs new file mode 100644 index 0000000..bc9f482 --- /dev/null +++ b/src/state.rs @@ -0,0 +1,38 @@ +// auto-imports +use crate::config::Config; + +// std +use std::{ + fs::{read_to_string, write}, + error::Error, +}; + +// log +use log::debug; + +/// Reads last tweet id from a file +pub fn read_state(s: &str) -> Option { + let state = read_to_string(s); + + if let Ok(s) = state { + debug!("Last Tweet ID (from file): {}", &s); + return s.parse::().ok(); + } + + None +} + +/// Writes last treated tweet id to a file +pub fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> { + write(f, format!("{}", s)) +} + +/********* + * Main functions +*********/ +/// Initiates the DB from path +pub fn init_db(config: &Config) -> Result<(), Box> { + println!("config.scootaloo.db_path: {}", config.scootaloo.db_path); + Ok(()) +} + diff --git a/src/twitter.rs b/src/twitter.rs new file mode 100644 index 0000000..f44c071 --- /dev/null +++ b/src/twitter.rs @@ -0,0 +1,83 @@ +// auto-imports +use crate::ScootalooError; +use crate::config::TwitterConfig; +use crate::util::cache_media; + +// std +use std::{ + error::Error, + collections::HashMap, +}; + +// egg-mode +use egg_mode::{ + Token, + KeyPair, + entities::{UrlEntity, MediaEntity, MediaType}, + user::UserID, + tweet::{ + Tweet, + user_timeline, + }, +}; + +/// Gets Twitter oauth2 token +pub fn get_oauth2_token(config: &TwitterConfig) -> Token { + let con_token = KeyPair::new(String::from(&config.consumer_key), String::from(&config.consumer_secret)); + let access_token = KeyPair::new(String::from(&config.access_key), String::from(&config.access_secret)); + + Token::Access { + consumer: con_token, + access: access_token, + } +} + +/// Gets Twitter user timeline +pub async fn get_user_timeline(config: &TwitterConfig, token: Token, lid: Option) -> Result, Box> { + // fix the page size to 200 as it is the maximum Twitter authorizes + let (_, feed) = user_timeline(UserID::from(String::from(&config.username)), true, false, &token) + .with_page_size(200) + .older(lid) + .await?; + + Ok(feed.to_vec()) +} + +/// Decodes urls from UrlEntities +pub fn decode_urls(urls: &Vec) -> HashMap { + let mut decoded_urls = HashMap::new(); + + for url in urls { + if url.expanded_url.is_some() { + // unwrap is safe here as we just verified that there is something inside expanded_url + decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap())); + } + } + + decoded_urls +} + +/// Retrieves a single media from a tweet and store it in a temporary file +pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result> { + match m.media_type { + MediaType::Photo => { + return cache_media(&m.media_url_https, t).await; + }, + _ => { + match &m.video_info { + Some(v) => { + for variant in &v.variants { + if variant.content_type == "video/mp4" { + return cache_media(&variant.url, t).await; + } + } + return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into()); + }, + None => { + return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into()); + }, + } + }, + }; +} + diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..6f7e838 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,37 @@ +// std +use std::error::Error; +use crate::ScootalooError; + +// reqwest +use reqwest::Url; + +// tokio +use tokio::{ + io::copy, + fs::{File, create_dir_all}, +}; + +/// Gets and caches Twitter Media inside the determined temp dir +pub async fn cache_media(u: &str, t: &str) -> Result> { + // create dir + create_dir_all(t).await?; + + // get file + let mut response = reqwest::get(u).await?; + + // create local file + let url = Url::parse(u)?; + let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))? + .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?; + + let dest_filepath = format!("{}/{}", t, dest_filename); + + let mut dest_file = File::create(&dest_filepath).await?; + + while let Some(chunk) = response.chunk().await? { + copy(&mut &*chunk, &mut dest_file).await?; + } + + Ok(dest_filepath) +} +