Merge branch 'noasync' into 'master'

getting async in a reasonnable way

See merge request veretcle/scootaloo!12

.gitlab-ci.yml

@@ -1,5 +1,13 @@
----
-include:
-  project: 'veretcle/ci-common'
-  ref: 'main'
-  file: 'ci_rust.yml'
+stages:
+      - build
+
+rust-latest:
+    stage: build
+    artifacts:
+      paths:
+        - target/release/scootaloo
+    image: rust:latest
+    script:
+        - cargo build --release --verbose
+        - strip target/release/${CI_PROJECT_NAME}
+

Cargo.toml

@@ -1,30 +1,21 @@
 [package]
 name = "scootaloo"
-version = "1.1.2"
+version = "0.4.0"
 authors = ["VC <veretcle+framagit@mateu.be>"]
-edition = "2021"
+edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-base64 = "^0.13"
-regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^4"
-egg-mode = "^0.16"
-rusqlite = "^0.27"
-isolang = "^2"
-tokio = { version = "^1", features = ["rt"]}
+clap = "^2.33"
 futures = "^0.3"
-megalodon = "^0.2"
-html-escape = "^0.2"
+egg-mode = { git = "https://github.com/egg-mode-rs/egg-mode", rev = "6b81073eba9c3b123ca0e80bdb5ef61d1758f131" }
+tokio = { version = "1", features = ["full"]}
+elefren = "^0.20"
+htmlescape = "^0.3"
 reqwest = "^0.11"
 log = "^0.4"
-simple_logger = "^2.1"
-mime = "^0.3"
+simple_logger = "^1.11"
 
-[profile.release]
-strip = true
-lto = true
-codegen-units = 1

README.md

@@ -7,61 +7,37 @@ It:
 
 If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
 
-RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
+RT are excluded, replies are included.but only the source threads are copied, not the actual replies to other Twitter users.
 
 # Usage
-## Configuring
 
 First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this:
 
-```toml
+```
 [scootaloo]
-db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
-cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
-rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneously
-## optional, this should be omitted the majority of the time
-## sometimes, twitter try to use french inclusive writting, but instead of using `·` (median point), they’re using `.`
-## this makes twitter interpret it as a URL, which is wrong
-## this parameter allows you to catch such URLs and apply the `display_url` (i.e. `tout.es`) instead of the `expanded_url` (i.e. `http://tout.es`)
-## in those particular cases
-## (!) use with caution, it might have some undesired effects
-show_url_as_display_url_for = "^http(s)://(.+)\\.es$"
-## optional, this allows you to replace the host for popular services such as YouTube of Twitter, or any other
-## with their more freely accessible equivalent
-[scootaloo.alternative_services_for]
-"tamere.lol" = "tonpere.mdr" ## quotes are necessary for both parameters
-"you.pi" = "you.pla"
-"www.you.pi" = "you.pla" ## this is an exact match, so you’ll need to lay out all the possibilities
+
+last_tweet_path="/usr/local/etc/last_tweet" ## file containing the last tweet id received, must be writable
+cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
 
 [twitter]
+username="NintendojoFR" ## User Timeline to copy
+
 ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
-page_size = 20 ## optional, default 200, max number of tweet retrieved
-consumer_key = "MYCONSUMERKEY"
-consumer_secret = "MYCONSUMERSECRET"
-access_key = "MYACCESSKEY"
-access_secret = "MYACCESSSECRET"
-
-[mastodon]
+consumer_key="MYCONSUMERKEY"
+consumer_secret="MYCONSUMERSECRET"
+access_key="MYACCESSKEY"
+access_secret="MYACCESSSECRET"
 ```
 
-Then run the command with the `init` subcommand to initiate the DB:
-```sh
-scootaloo init
-```
-
-This subcommand is completely idempotent.
-
 Then run the command with the `register` subcommand:
-```sh
+```
 scootaloo register --host https://m.nintendojo.fr
 ```
 
 This will give you the end of the TOML file. It will look like this:
 
-```toml
-[mastodon.nintendojofr] ## account
-twitter_screen_name = "NintendojoFR" ## User Timeline to copy
-mastodon_screen_name = "nintendojofr" ## optional, Mastodon account name used for smart mentions
+```
+[mastodon]
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
 client_secret = "MYCLIENTSECRET"
@@ -69,20 +45,9 @@ redirect = "urn:ietf:wg:oauth:2.0:oob"
 token = "MYTOKEN"
 ```
 
-You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
-
-You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so:
-```
-twitter_page_size = 40
-```
-
-## Running
-
 You can then run the application via `cron` for example. Here is the generic usage:
 
-```sh
-A Twitter to Mastodon bot
-
+```
 USAGE:
     scootaloo [OPTIONS] [SUBCOMMAND]
 
@@ -92,38 +57,20 @@ FLAGS:
 
 OPTIONS:
     -c, --config <CONFIG_FILE>    TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)
-    -l, --loglevel <LOGLEVEL>     Log level. Valid values are: Off, Warn, Error, Info, Debug
 
 SUBCOMMANDS:
     help        Prints this message or the help of the given subcommand(s)
-    init        Command to init Scootaloo DB
-    migrate     Command to migrate Scootaloo DB
     register    Command to register to a Mastodon Instance
 ```
 
 # Quirks
 
-Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the DB file before copying an old account.
+Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the `last_tweet` file before copying an old account.
 
-You can insert that Tweet number, by connecting to the DB you created:
-```sh
-sqlite3 /var/lib/scootaloo/scootaloo.sqlite
+You can do that with a command like:
+```
+echo -n '8189881949849' > last_tweet
 ```
 
-And inserting the data:
+**This file should only contain the last tweet ID without any other char (no EOL or new line).**
 
-```sql
-INSERT INTO tweet_to_toot VALUES ("<twitter_screen_name>", 1383782580412030982, "<twitter_screen_name>");
-```
-
-The last value is supposed to be the Toot ID. It cannot be null, so you better initialize it with something unique, like the Twitter Screen Name for example.
-
-# Migrating from Scootaloo ⩽ 0.6.1
-
-The DB scheme has change between version 0.6.x and 0.7.x (this is due to the multi-account nature of Scootaloo from 0.7.x onward). You need to migrate your DB. You can do so by issuing the command:
-
-```
-scootaloo migrate
-```
-
-You can optionnally specify a screen name with the `--name` option. By default, it’ll take the first screen name in the config file.

src/config.rs

@@ -1,52 +0,0 @@
-use std::{collections::HashMap, fs::read_to_string};
-
-use serde::Deserialize;
-
-/// General configuration Struct
-#[derive(Debug, Deserialize)]
-pub struct Config {
-    pub twitter: TwitterConfig,
-    pub mastodon: HashMap<String, MastodonConfig>,
-    pub scootaloo: ScootalooConfig,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct TwitterConfig {
-    pub consumer_key: String,
-    pub consumer_secret: String,
-    pub access_key: String,
-    pub access_secret: String,
-    pub page_size: Option<i32>,
-}
-
-#[derive(Debug, Deserialize, Clone)]
-pub struct MastodonConfig {
-    pub twitter_screen_name: String,
-    pub mastodon_screen_name: Option<String>,
-    pub twitter_page_size: Option<i32>,
-    pub base: String,
-    pub client_id: String,
-    pub client_secret: String,
-    pub redirect: String,
-    pub token: String,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct ScootalooConfig {
-    pub db_path: String,
-    pub cache_path: String,
-    pub rate_limit: Option<usize>,
-    pub show_url_as_display_url_for: Option<String>,
-    pub alternative_services_for: Option<HashMap<String, String>>,
-}
-
-/// Parses the TOML file into a Config Struct
-pub fn parse_toml(toml_file: &str) -> Config {
-    let toml_config = read_to_string(toml_file)
-        .unwrap_or_else(|e| panic!("Cannot open config file {}: {}", toml_file, e));
-
-    let config: Config = toml::from_str(&toml_config)
-        .unwrap_or_else(|e| panic!("Cannot parse TOML file {}: {}", toml_file, e));
-
-    config
-}

src/error.rs

@@ -1,41 +0,0 @@
-use std::{
-    boxed::Box,
-    convert::From,
-    error::Error,
-    fmt::{Display, Formatter, Result},
-};
-
-use megalodon::error::Error as megalodonError;
-
-#[derive(Debug)]
-pub struct ScootalooError {
-    details: String,
-}
-
-impl ScootalooError {
-    pub fn new(msg: &str) -> ScootalooError {
-        ScootalooError {
-            details: msg.to_string(),
-        }
-    }
-}
-
-impl Error for ScootalooError {}
-
-impl Display for ScootalooError {
-    fn fmt(&self, f: &mut Formatter) -> Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl From<Box<dyn Error>> for ScootalooError {
-    fn from(error: Box<dyn Error>) -> Self {
-        ScootalooError::new(&format!("Error in a subset crate: {}", error))
-    }
-}
-
-impl From<megalodonError> for ScootalooError {
-    fn from(error: megalodonError) -> Self {
-        ScootalooError::new(&format!("Error in megalodon crate: {}", error))
-    }
-}

src/lib.rs

@@ -1,317 +1,428 @@
-mod error;
-use error::ScootalooError;
-
-mod config;
-pub use config::parse_toml;
-use config::Config;
-
-mod mastodon;
-pub use mastodon::register;
-use mastodon::*;
-
-mod twitter;
-use twitter::*;
-
-mod util;
-use util::{base64_media, generate_media_ids};
-
-mod state;
-pub use state::{init_db, migrate_db};
-use state::{read_state, write_state, TweetToToot};
-
-use futures::StreamExt;
-use html_escape::decode_html_entities;
-use isolang::Language;
-use log::info;
-use megalodon::{
-    megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
+// std
+use std::{
+    borrow::Cow,
+    collections::HashMap,
+    io::stdin,
+    fmt,
+    fs::{read_to_string, write},
+    error::Error,
 };
-use regex::Regex;
-use rusqlite::Connection;
-use std::sync::Arc;
-use tokio::{spawn, sync::Mutex};
 
-const DEFAULT_RATE_LIMIT: usize = 4;
-const DEFAULT_PAGE_SIZE: i32 = 200;
+// toml
+use serde::Deserialize;
+
+// egg-mode
+use egg_mode::{
+    Token,
+    KeyPair,
+    entities::{UrlEntity, MediaEntity, MentionEntity, MediaType},
+    user::UserID,
+    tweet::{
+        Tweet,
+        user_timeline,
+    },
+};
+
+// elefren
+use elefren::{
+    prelude::*,
+    apps::App,
+    status_builder::StatusBuilder,
+    scopes::Scopes,
+};
+
+// reqwest
+use reqwest::Url;
+
+// tokio
+use tokio::{
+    io::copy,
+    fs::{File, create_dir_all, remove_file},
+};
+
+// htmlescape
+use htmlescape::decode_html;
+
+// log
+use log::{info, warn, error, debug};
+
+/**********
+ * Generic usage functions
+***********/
+/*
+ * Those functions are related to the Twitter side of things
+ */
+/// Reads last tweet id from a file
+fn read_state(s: &str) -> Option<u64> {
+    let state = read_to_string(s);
+
+    if let Ok(s) = state {
+        debug!("Last Tweet ID (from file): {}", &s);
+        return s.parse::<u64>().ok();
+    }
+
+    None
+}
+
+/// Writes last treated tweet id to a file
+fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
+    write(f, format!("{}", s))
+}
+
+/// Gets Twitter oauth2 token
+fn get_oauth2_token(config: &Config) -> Token {
+    let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
+    let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
+
+    Token::Access {
+        consumer: con_token,
+        access: access_token,
+    }
+}
+
+/// Gets Twitter user timeline
+async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
+    // fix the page size to 200 as it is the maximum Twitter authorizes
+    let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
+                .with_page_size(200)
+                .older(lid)
+                .await?;
+
+    Ok(feed.to_vec())
+}
+
+/// Decodes urls from UrlEntities
+fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
+    let mut decoded_urls = HashMap::new();
+
+    for url in urls {
+        if url.expanded_url.is_some() {
+            // unwrap is safe here as we just verified that there is something inside expanded_url
+            decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
+        }
+    }
+
+    decoded_urls
+}
+
+/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
+/// Fediverse
+fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
+    let mut decoded_mentions = HashMap::new();
+
+    for um in ums {
+        decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
+    }
+
+    decoded_mentions
+}
+
+/// Retrieves a single media from a tweet and store it in a temporary file
+async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
+    match m.media_type {
+        MediaType::Photo => {
+            return cache_media(&m.media_url_https, t).await;
+        },
+        _ => {
+            match &m.video_info {
+                Some(v) => {
+                    for variant in &v.variants {
+                        if variant.content_type == "video/mp4" {
+                            return cache_media(&variant.url, t).await;
+                        }
+                    }
+                    return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
+                },
+                None => {
+                    return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
+                },
+            }
+        },
+    };
+}
+
+/*
+ * Those functions are related to the Mastodon side of things
+ */
+/// Gets Mastodon Data
+fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
+    let data = Data {
+        base: Cow::from(String::from(&masto.base)),
+        client_id: Cow::from(String::from(&masto.client_id)),
+        client_secret: Cow::from(String::from(&masto.client_secret)),
+        redirect: Cow::from(String::from(&masto.redirect)),
+        token: Cow::from(String::from(&masto.token)),
+    };
+
+    Mastodon::from(data)
+}
+
+/// Builds toot text from tweet
+fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
+    let mut toot = String::from(&tweet.text);
+
+    let decoded_urls = decode_urls(&tweet.entities.urls);
+
+    for decoded_url in decoded_urls {
+        toot = toot.replace(&decoded_url.0, &decoded_url.1);
+    }
+
+    let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
+
+    for decoded_mention in decoded_mentions {
+        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
+    }
+
+    if let Ok(t) = decode_html(&toot) {
+        toot = t;
+    }
+
+    Ok(toot)
+}
+
+/*
+ * Generic private functions
+ */
+/// Gets and caches Twitter Media inside the determined temp dir
+async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
+    // create dir
+    create_dir_all(t).await?;
+
+    // get file
+    let mut response = reqwest::get(u).await?;
+
+    // create local file
+    let url = Url::parse(u)?;
+    let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
+                    .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
+
+    let dest_filepath = format!("{}/{}", t, dest_filename);
+
+    let mut dest_file = File::create(&dest_filepath).await?;
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut dest_file).await?;
+    }
+
+    Ok(dest_filepath)
+}
+
+/**********
+ * This is the struct that holds the Mastodon Media ID and the Twitter Media URL at the same Time
+**********/
+#[derive(Debug)]
+struct ScootalooSpawnResponse {
+    mastodon_media_id: String,
+    twitter_media_url: String,
+}
+
+/**********
+ * local error handler
+**********/
+#[derive(Debug)]
+struct ScootalooError {
+    details: String,
+}
+
+impl ScootalooError {
+    fn new(msg: &str) -> ScootalooError {
+        ScootalooError {
+            details: String::from(msg),
+        }
+    }
+}
+
+impl fmt::Display for ScootalooError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.details)
+    }
+}
+
+impl std::error::Error for ScootalooError {
+    fn description(&self) -> &str {
+        &self.details
+    }
+}
+
+/**********
+ * Config structure
+***********/
+/// General configuration Struct
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    twitter: TwitterConfig,
+    mastodon: MastodonConfig,
+    scootaloo: ScootalooConfig,
+}
+
+#[derive(Debug, Deserialize)]
+struct TwitterConfig {
+    username: String,
+    consumer_key: String,
+    consumer_secret: String,
+    access_key: String,
+    access_secret: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct MastodonConfig {
+    base: String,
+    client_id: String,
+    client_secret: String,
+    redirect: String,
+    token: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct ScootalooConfig {
+    last_tweet_path: String,
+    cache_path: String,
+}
+
+/*********
+ * Main functions
+*********/
+/// Parses the TOML file into a Config Struct
+pub fn parse_toml(toml_file: &str) -> Config {
+    let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
+        panic!("Cannot open config file {}: {}", toml_file, e)
+    );
+
+    let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
+        panic!("Cannot parse TOML file {}: {}", toml_file, e)
+    );
+
+    config
+}
+
+/// Generic register function
+/// As this function is supposed to be run only once, it will panic for every error it encounters
+/// Most of this function is a direct copy/paste of the official `elefren` crate
+pub fn register(host: &str) {
+    let mut builder = App::builder();
+    builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
+        .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
+        .scopes(Scopes::write_all())
+        .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
+
+    let app = builder.build().expect("Cannot build the app");
+
+    let registration = Registration::new(host).register(app).expect("Cannot build registration object");
+    let url = registration.authorize_url().expect("Cannot generate registration URI!");
+
+    println!("Click this link to authorize on Mastodon: {}", url);
+    println!("Paste the returned authorization code: ");
+
+    let mut input = String::new();
+    stdin().read_line(&mut input).expect("Unable to read back registration code!");
+
+    let code = input.trim();
+    let mastodon = registration.complete(code).expect("Unable to create access token!");
+
+    let toml = toml::to_string(&*mastodon).unwrap();
+
+    println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
+}
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
-    // open the SQLite connection
-    let conn = Arc::new(Mutex::new(
-        Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
-            panic!(
-                "Something went wrong when opening the DB {}: {}",
-                &config.scootaloo.db_path, e
-            )
-        }),
-    ));
+    // retrieve the last tweet ID for the username
+    let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
 
-    let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
+    // get OAuth2 token
+    let token = get_oauth2_token(&config);
 
-    let display_url_re = config
-        .scootaloo
-        .show_url_as_display_url_for
-        .as_ref()
-        .map(|r|
-        // we want to panic in case the RE is not valid
-        Regex::new(r).unwrap());
+    // get Mastodon instance
+    let mastodon = get_mastodon_token(&config.mastodon);
 
-    let mut stream = futures::stream::iter(config.mastodon.into_values())
-        .map(|mastodon_config| {
-            // calculate Twitter page size
-            let page_size = mastodon_config
-                .twitter_page_size
-                .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+    // get user timeline feed (Vec<tweet>)
+    let mut feed = get_user_timeline(&config, token, last_tweet_id)
+        .await
+        .unwrap_or_else(|e|
+        panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
+    );
 
-            // create temporary value for each task
-            let scootaloo_cache_path = config.scootaloo.cache_path.clone();
-            let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
-            let display_url_re = display_url_re.clone();
-            let token = get_oauth2_token(&config.twitter);
-            let task_conn = conn.clone();
-            let global_mastodon_config = global_mastodon_config.clone();
+    // empty feed -> exiting
+    if feed.is_empty() {
+        info!("Nothing to retrieve since last time, exiting…");
+        return;
+    }
 
-            spawn(async move {
-                info!("Starting treating {}", &mastodon_config.twitter_screen_name);
-                // retrieve the last tweet ID for the username
-                let lconn = task_conn.lock().await;
-                let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
-                    .map(|r| r.tweet_id);
-                drop(lconn);
+    // order needs to be chronological
+    feed.reverse();
 
-                // get reversed, curated user timeline
-                let feed = get_user_timeline(
-                    &mastodon_config.twitter_screen_name,
-                    &token,
-                    last_tweet_id,
-                    page_size,
-                )
-                .await?;
-
-                // get Mastodon instance
-                let mastodon = get_mastodon_token(&mastodon_config);
-
-                for tweet in &feed {
-                    info!("Treating Tweet {} inside feed", tweet.id);
-
-                    // basic toot text
-                    let mut status_text = tweet.text.clone();
-
-                    // add mentions and smart mentions
-                    if !&tweet.entities.user_mentions.is_empty() {
-                        info!("Tweet contains mentions, add them!");
-                        let global_mastodon_config = global_mastodon_config.lock().await;
-                        twitter_mentions(
-                            &mut status_text,
-                            &tweet.entities.user_mentions,
-                            &global_mastodon_config,
-                        );
-                        drop(global_mastodon_config);
-                    }
-
-                    if !&tweet.entities.urls.is_empty() {
-                        info!("Tweet contains links, add them!");
-                        let mut associated_urls =
-                            associate_urls(&tweet.entities.urls, &display_url_re);
-
-                        if let Some(q) = &tweet.quoted_status {
-                            if let Some(u) = &q.user {
-                                info!(
-                                    "Tweet {} contains a quote, we try to find it within the DB",
-                                    tweet.id
-                                );
-                                // we know we have a quote and a user, we can lock both the
-                                // connection to DB and global_config
-                                // we will release them manually as soon as they’re useless
-                                let lconn = task_conn.lock().await;
-                                let global_mastodon_config = global_mastodon_config.lock().await;
-                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
-                                {
-                                    info!("We have found the associated toot({})", &r.toot_id);
-                                    // drop conn immediately after the request: we won’t need it
-                                    // any more and the treatment there might be time-consuming
-                                    drop(lconn);
-                                    if let Some((m, t)) =
-                                        find_mastodon_screen_name_by_twitter_screen_name(
-                                            &r.twitter_screen_name,
-                                            &global_mastodon_config,
-                                        )
-                                    {
-                                        // drop the global conf, we have all we required, no need
-                                        // to block it further
-                                        drop(global_mastodon_config);
-                                        replace_tweet_by_toot(
-                                            &mut associated_urls,
-                                            &r.twitter_screen_name,
-                                            q.id,
-                                            &m,
-                                            &t,
-                                            &r.toot_id,
-                                        );
-                                    }
-                                }
-                            }
-                        }
-
-                        if let Some(a) = &scootaloo_alt_services {
-                            replace_alt_services(&mut associated_urls, a);
-                        }
-
-                        decode_urls(&mut status_text, &associated_urls);
-                    }
-
-                    // building associative media list
-                    let (media_url, status_medias) =
-                        generate_media_ids(tweet, &scootaloo_cache_path, &mastodon).await;
-
-                    status_text = status_text.replace(&media_url, "");
-
-                    // now that the text won’t be altered anymore, we can safely remove HTML
-                    // entities
-                    status_text = decode_html_entities(&status_text).to_string();
-
-                    info!("Building corresponding Mastodon status");
-
-                    let mut post_status = PostStatusInputOptions {
-                        media_ids: None,
-                        poll: None,
-                        in_reply_to_id: None,
-                        sensitive: None,
-                        spoiler_text: None,
-                        visibility: None,
-                        scheduled_at: None,
-                        language: None,
-                        quote_id: None,
-                    };
-
-                    if !status_medias.is_empty() {
-                        post_status.media_ids = Some(status_medias);
-                    }
-
-                    // thread if necessary
-                    if tweet.in_reply_to_user_id.is_some() {
-                        let lconn = task_conn.lock().await;
-                        if let Ok(Some(r)) = read_state(
-                            &lconn,
-                            &mastodon_config.twitter_screen_name,
-                            tweet.in_reply_to_status_id,
-                        ) {
-                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
-                        }
-                        drop(lconn);
-                    }
-
-                    // language if any
-                    if let Some(l) = &tweet.lang {
-                        if let Some(r) = Language::from_639_1(l) {
-                            post_status.language = Some(r.to_string());
-                        }
-                    }
-
-                    // can be activated for test purposes
-                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
-
-                    let published_status = mastodon
-                        .post_status(status_text, Some(&post_status))
-                        .await?
-                        .json();
-                    // this will return if it cannot publish the status preventing the last_tweet from
-                    // being written into db
-
-                    let ttt_towrite = TweetToToot {
-                        twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
-                        tweet_id: tweet.id,
-                        toot_id: published_status.id,
-                    };
-
-                    // write the current state (tweet ID and toot ID) to avoid copying it another time
-                    let lconn = task_conn.lock().await;
-                    write_state(&lconn, ttt_towrite)?;
-                    drop(lconn);
+    for tweet in &feed {
+        debug!("Treating Tweet {} inside feed", tweet.id);
+        // determine if the tweet is part of a thread (response to self) or a standard response
+        if let Some(r) =  &tweet.in_reply_to_screen_name {
+                if &r.to_lowercase() != &config.twitter.username.to_lowercase() {
+                    // we are responding not threading
+                    info!("Tweet is a direct response, skipping");
+                    continue;
                 }
-                Ok::<(), ScootalooError>(())
-            })
-        })
-        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+        };
 
-    // launch and wait for every handle
-    while let Some(result) = stream.next().await {
-        match result {
-            Ok(Err(e)) => eprintln!("Error within thread: {}", e),
-            Err(e) => eprintln!("Error with thread: {}", e),
-            _ => (),
+        // build basic status by just yielding text and dereferencing contained urls
+        let mut status_text = match build_basic_status(tweet) {
+            Ok(t) => t,
+            Err(e) => {
+                error!("Could not create status from tweet {}: {}", tweet.id ,e);
+                continue;
+            },
+        };
+
+        let mut status_medias: Vec<String> = vec![];
+
+       // reupload the attachments if any
+        if let Some(m) = &tweet.extended_entities {
+            for media in &m.media {
+                let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path).await {
+                    Ok(m) => m,
+                    Err(e) => {
+                        error!("Cannot get tweet media for {}: {}", &media.url, e);
+                        continue;
+                    },
+                };
+
+                let mastodon_media_ids = match mastodon.media(Cow::from(String::from(&local_tweet_media_path))) {
+                    Ok(m) => {
+                        remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
+                            warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
+                        );
+                        m.id
+                    },
+                    Err(e) => {
+                        error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
+                        continue;
+                    }
+                };
+
+                status_medias.push(mastodon_media_ids);
+
+                // last step, removing the reference to the media from with the toot’s text
+                status_text = status_text.replace(&media.url, "");
+            }
         }
+        // finished reuploading attachments, now let’s do the toot baby!
+
+        debug!("Building corresponding Mastodon status");
+        let status = StatusBuilder::new()
+            .status(&status_text)
+            .media_ids(status_medias)
+            .build()
+            .expect(&format!("Cannot build status with text {}", &status_text));
+
+        // publish status
+        // again unwrap is safe here as we are in the main thread
+        mastodon.new_status(status).unwrap();
+        // this will panic if it cannot publish the status, which is a good thing, it allows the
+        // last_tweet gathered not to be written
+
+        // write the current state (tweet ID) to avoid copying it another time
+        write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e|
+            panic!("Can’t write the last tweet retrieved: {}", e)
+        );
     }
 }
 
-/// Copies the Twitter profile into Mastodon
-#[tokio::main]
-pub async fn profile(config: Config, bot: Option<bool>) {
-    let mut stream = futures::stream::iter(config.mastodon.into_values())
-        .map(|mastodon_config| {
-            let token = get_oauth2_token(&config.twitter);
-
-            spawn(async move {
-                // get the user of the last tweet of the feed
-                let twitter_user =
-                    get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
-                        .await?
-                        .first()
-                        .ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
-                        .clone()
-                        .user
-                        .ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
-
-                let note = get_note_from_description(
-                    &twitter_user.description,
-                    &twitter_user.entities.description.urls,
-                );
-
-                // let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
-
-                let display_name = Some(String::from_utf16_lossy(
-                    &twitter_user
-                        .name
-                        .encode_utf16()
-                        .take(30)
-                        .collect::<Vec<u16>>(),
-                ));
-
-                let header = match twitter_user.profile_banner_url {
-                    Some(h) => Some(base64_media(&h).await?),
-                    None => None,
-                };
-
-                let update_creds = UpdateCredentialsInputOptions {
-                    discoverable: None,
-                    bot,
-                    display_name,
-                    note,
-                    avatar: Some(
-                        base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
-                            .await?,
-                    ),
-                    header,
-                    locked: None,
-                    source: None,
-                    fields_attributes: None,
-                };
-
-                let mastodon = get_mastodon_token(&mastodon_config);
-
-                mastodon.update_credentials(Some(&update_creds)).await?;
-
-                Ok::<(), ScootalooError>(())
-            })
-        })
-        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
-
-    while let Some(result) = stream.next().await {
-        match result {
-            Ok(Err(e)) => eprintln!("Error within thread: {}", e),
-            Err(e) => eprintln!("Error with thread: {}", e),
-            _ => (),
-        }
-    }
-}

src/main.rs

@@ -1,186 +1,63 @@
-use clap::{Arg, ArgAction, Command};
-use log::LevelFilter;
+// self
 use scootaloo::*;
+
+// clap
+use clap::{App, Arg, SubCommand};
+
+// log
+use log::{LevelFilter, error};
 use simple_logger::SimpleLogger;
+
+// std
 use std::str::FromStr;
 
-const DEFAULT_CONFIG_PATH: &str = "/usr/local/etc/scootaloo.toml";
-
 fn main() {
-    let matches = Command::new(env!("CARGO_PKG_NAME"))
-        .version(env!("CARGO_PKG_VERSION"))
-        .about("A Twitter to Mastodon bot")
-        .arg(
-            Arg::new("config")
-                .short('c')
-                .long("config")
-                .value_name("CONFIG_FILE")
-                .help("TOML config file for scootaloo")
-                .num_args(1)
-                .default_value(DEFAULT_CONFIG_PATH)
-                .display_order(1),
-        )
-        .arg(
-            Arg::new("log_level")
-                .short('l')
-                .long("loglevel")
-                .value_name("LOGLEVEL")
-                .help("Log level")
-                .num_args(1)
-                .value_parser(["Off", "Warn", "Error", "Info", "Debug"])
-                .display_order(2),
-        )
-        .subcommand(
-            Command::new("register")
-                .version(env!("CARGO_PKG_VERSION"))
-                .about("Command to register to a Mastodon Instance")
-                .arg(
-                    Arg::new("host")
-                        .short('H')
-                        .long("host")
-                        .value_name("HOST")
-                        .help("Base URL of the Mastodon instance to register to (no default)")
-                        .num_args(1)
-                        .required(true)
-                        .display_order(1)
-                    )
-                .arg(
-                    Arg::new("name")
-                        .short('n')
-                        .long("name")
-                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .num_args(1)
-                        .required(true)
-                        .display_order(2)
-                ),
-        )
-        .subcommand(
-            Command::new("init")
-                .version(env!("CARGO_PKG_VERSION"))
-                .about("Command to init Scootaloo DB")
-                .arg(
-                    Arg::new("config")
-                        .short('c')
-                        .long("config")
-                        .value_name("CONFIG_FILE")
-                        .help(&format!(
-                            "TOML config file for scootaloo (default {})",
-                            DEFAULT_CONFIG_PATH
-                        ))
-                        .default_value(DEFAULT_CONFIG_PATH)
-                        .num_args(1)
-                        .display_order(1),
-                ),
-        )
-        .subcommand(
-            Command::new("migrate")
-            .version(env!("CARGO_PKG_VERSION"))
-            .about("Command to migrate Scootaloo DB")
-            .arg(
-                Arg::new("config")
-                    .short('c')
-                    .long("config")
-                    .value_name("CONFIG_FILE")
-                    .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
-                    .default_value(DEFAULT_CONFIG_PATH)
-                    .num_args(1)
-                    .display_order(1),
-            )
-            .arg(
-                Arg::new("name")
-                    .short('n')
-                    .long("name")
-                    .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                    .num_args(1)
-                    .display_order(2)
-            )
-        )
-        .subcommand(
-            Command::new("copyprofile")
-            .version(env!("CARGO_PKG_VERSION"))
-            .about("Command to copy a Twitter profile into Mastodon")
-            .arg(
-                Arg::new("config")
-                    .short('c')
-                    .long("config")
-                    .value_name("CONFIG_FILE")
-                    .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
-                    .default_value(DEFAULT_CONFIG_PATH)
-                    .num_args(1)
-                    .display_order(1),
-            )
-            .arg(
-                Arg::new("name")
-                    .short('n')
-                    .long("name")
-                    .help("Mastodon Config name (as seen in the config file)")
-                    .num_args(1)
-                    .display_order(2)
-            )
-            .arg(
-                Arg::new("bot")
-                    .short('b')
-                    .long("bot")
-                    .help("Declare user as bot")
-                    .action(ArgAction::SetTrue)
-                    .display_order(3)
-            )
-        )
-        .get_matches();
+    let matches = App::new(env!("CARGO_PKG_NAME"))
+                        .version(env!("CARGO_PKG_VERSION"))
+                        .about("A Twitter to Mastodon bot")
+                        .arg(Arg::with_name("config")
+                             .short("c")
+                             .long("config")
+                             .value_name("CONFIG_FILE")
+                             .help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)")
+                             .takes_value(true)
+                             .display_order(1))
+                        .arg(Arg::with_name("log_level")
+                             .short("l")
+                             .long("loglevel")
+                             .value_name("LOGLEVEL")
+                             .help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
+                             .takes_value(true)
+                             .display_order(2))
+                             .subcommand(SubCommand::with_name("register")
+                                      .version(env!("CARGO_PKG_VERSION"))
+                                      .about("Command to register to a Mastodon Instance")
+                                      .arg(Arg::with_name("host")
+                                           .short("H")
+                                           .long("host")
+                                           .value_name("HOST")
+                                           .help("Base URL of the Mastodon instance to register to (no default)")
+                                           .takes_value(true)
+                                           .required(true)
+                                           .display_order(1)))
+                        .get_matches();
+    if let Some(matches) = matches.subcommand_matches("register") {
+        register(matches.value_of("host").unwrap());
+        return;
+    }
 
-    match matches.subcommand() {
-        Some(("register", sub_m)) => {
-            register(
-                sub_m.get_one::<String>("host").unwrap(),
-                sub_m.get_one::<String>("name").unwrap(),
-            );
-            return;
-        }
-        Some(("init", sub_m)) => {
-            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
-            init_db(&config.scootaloo.db_path).unwrap();
-            return;
-        }
-        Some(("migrate", sub_m)) => {
-            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
-            let config_twitter_screen_name =
-                &config.mastodon.values().next().unwrap().twitter_screen_name;
-            migrate_db(
-                &config.scootaloo.db_path,
-                sub_m
-                    .get_one::<String>("name")
-                    .unwrap_or(config_twitter_screen_name),
-            )
-            .unwrap();
-            return;
-        }
-        Some(("copyprofile", sub_m)) => {
-            let mut config = parse_toml(sub_m.get_one::<String>("config").unwrap());
-            // filters out the user passed in cli from the global configuration
-            if let Some(m) = sub_m.get_one::<String>("name") {
-                match config.mastodon.get(m) {
-                    Some(_) => {
-                        config.mastodon.retain(|k, _| k == m);
-                    }
-                    None => panic!("Config file does not contain conf for {}", &m),
-                }
+    if matches.is_present("log_level") {
+        match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
+            Ok(level) => { SimpleLogger::new().with_level(level).init().unwrap()},
+            Err(e) => {
+                SimpleLogger::new().with_level(LevelFilter::Error).init().unwrap();
+                error!("Unknown log level filter: {}", e);
             }
-
-            profile(config, sub_m.get_flag("bot").then_some(true));
-
-            return;
-        }
-        _ => (),
+        };
     }
 
-    if let Some(level) = matches.get_one::<String>("log_level") {
-        SimpleLogger::new()
-            .with_level(LevelFilter::from_str(level).unwrap())
-            .init()
-            .unwrap();
-    }
-
-    let config = parse_toml(matches.get_one::<String>("config").unwrap());
+    let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
 
     run(config);
 }
+

src/mastodon.rs

@@ -1,583 +0,0 @@
-use crate::config::MastodonConfig;
-
-use egg_mode::{
-    entities::{MentionEntity, UrlEntity},
-    user::UserEntityDetail,
-};
-use megalodon::{
-    generator,
-    mastodon::Mastodon,
-    megalodon::{AppInputOptions, CredentialsFieldAttribute},
-};
-use regex::Regex;
-use std::{collections::HashMap, io::stdin};
-
-/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
-/// as well
-pub fn twitter_mentions(
-    toot: &mut String,
-    ums: &[MentionEntity],
-    masto: &HashMap<String, MastodonConfig>,
-) {
-    let tm: HashMap<String, String> = ums
-        .iter()
-        .map(|s| {
-            (
-                format!("@{}", s.screen_name),
-                format!("@{}@twitter.com", s.screen_name),
-            )
-        })
-        .chain(
-            masto
-                .values()
-                .filter(|s| s.mastodon_screen_name.is_some())
-                .map(|s| {
-                    (
-                        format!("@{}", s.twitter_screen_name),
-                        format!(
-                            "@{}@{}",
-                            s.mastodon_screen_name.as_ref().unwrap(),
-                            s.base.split('/').last().unwrap()
-                        ),
-                    )
-                })
-                .collect::<HashMap<String, String>>(),
-        )
-        .collect();
-
-    for (k, v) in tm {
-        *toot = toot.replace(&k, &v);
-    }
-}
-
-/// Decodes urls in toot
-pub fn decode_urls(toot: &mut String, urls: &HashMap<String, String>) {
-    for (k, v) in urls {
-        *toot = toot.replace(k, v);
-    }
-}
-
-/// Reassociates source url with destination url for rewritting
-/// this takes a Teet UrlEntity and an optional Regex
-pub fn associate_urls(urls: &[UrlEntity], re: &Option<Regex>) -> HashMap<String, String> {
-    urls.iter()
-        .filter(|s| s.expanded_url.is_some())
-        .map(|s| {
-            (s.url.to_owned(), {
-                let mut def = s.expanded_url.as_deref().unwrap().to_owned();
-
-                if let Some(r) = re {
-                    if r.is_match(s.expanded_url.as_deref().unwrap()) {
-                        def = s.display_url.to_owned();
-                    }
-                }
-
-                def
-            })
-        })
-        .collect::<HashMap<String, String>>()
-}
-
-/// Replaces the commonly used services by mirrors, if asked to
-pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<String, String>) {
-    for val in urls.values_mut() {
-        for (k, v) in alts {
-            *val = val.replace(&format!("/{}/", k), &format!("/{}/", v));
-        }
-    }
-}
-
-/// Finds a Mastodon screen_name/base_url from a MastodonConfig
-pub fn find_mastodon_screen_name_by_twitter_screen_name(
-    twitter_screen_name: &str,
-    masto: &HashMap<String, MastodonConfig>,
-) -> Option<(String, String)> {
-    masto.iter().find_map(|(_, v)| {
-        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
-            Some((
-                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
-                v.base.to_owned(),
-            ))
-        } else {
-            None
-        }
-    })
-}
-
-/// Replaces the original quoted tweet by the corresponding toot
-pub fn replace_tweet_by_toot(
-    urls: &mut HashMap<String, String>,
-    twitter_screen_name: &str,
-    tweet_id: u64,
-    mastodon_screen_name: &str,
-    base_url: &str,
-    toot_id: &str,
-) {
-    for val in urls.values_mut() {
-        if val.to_lowercase().starts_with(&format!(
-            "https://twitter.com/{}/status/{}",
-            twitter_screen_name.to_lowercase(),
-            tweet_id
-        )) {
-            *val = format!("{}/@{}/{}", base_url, mastodon_screen_name, toot_id);
-        }
-    }
-}
-
-/// Gets Mastodon Data
-pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
-}
-
-/// Gets note from twitter_user description
-pub fn get_note_from_description(t: &Option<String>, urls: &[UrlEntity]) -> Option<String> {
-    t.as_ref().map(|d| {
-        let mut n = d.to_owned();
-        let a_urls = associate_urls(urls, &None);
-        decode_urls(&mut n, &a_urls);
-        n
-    })
-}
-
-/// Gets fields_attribute from UserEntityDetail
-#[allow(dead_code)]
-pub fn get_attribute_from_url(
-    user_entity_detail: &Option<UserEntityDetail>,
-) -> Option<Vec<CredentialsFieldAttribute>> {
-    user_entity_detail.as_ref().and_then(|u| {
-        u.urls.first().and_then(|v| {
-            v.expanded_url.as_ref().map(|e| {
-                vec![CredentialsFieldAttribute {
-                    name: v.display_url.to_string(),
-                    value: e.to_string(),
-                }]
-            })
-        })
-    })
-}
-
-/// Generic register function
-/// As this function is supposed to be run only once, it will panic for every error it encounters
-/// Most of this function is a direct copy/paste of the official `elefren` crate
-#[tokio::main]
-pub async fn register(host: &str, screen_name: &str) {
-    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
-
-    let options = AppInputOptions {
-        redirect_uris: None,
-        scopes: Some(
-            [
-                "read:accounts".to_string(),
-                "write:accounts".to_string(),
-                "write:media".to_string(),
-                "write:statuses".to_string(),
-            ]
-            .to_vec(),
-        ),
-        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
-    };
-
-    let app_data = mastodon
-        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
-        .await
-        .expect("Cannot build registration object!");
-
-    let url = app_data.url.expect("Cannot generate registration URI!");
-
-    println!("Click this link to authorize on Mastodon: {}", url);
-    println!("Paste the returned authorization code: ");
-
-    let mut input = String::new();
-    stdin()
-        .read_line(&mut input)
-        .expect("Unable to read back registration code!");
-
-    let token_data = mastodon
-        .fetch_access_token(
-            app_data.client_id.to_owned(),
-            app_data.client_secret.to_owned(),
-            input.trim().to_string(),
-            megalodon::default::NO_REDIRECT.to_string(),
-        )
-        .await
-        .expect("Unable to create access token!");
-
-    let mastodon = generator(
-        megalodon::SNS::Mastodon,
-        host.to_string(),
-        Some(token_data.access_token.to_owned()),
-        None,
-    );
-
-    let current_account = mastodon
-        .verify_account_credentials()
-        .await
-        .expect("Unable to access account information!")
-        .json();
-
-    println!(
-        r#"Please insert the following block at the end of your configuration file:
-[mastodon.{}]
-twitter_screen_name = "{}"
-mastodon_screen_name = "{}"
-base = "{}"
-client_id = "{}"
-client_secret = "{}"
-redirect = "{}"
-token = "{}""#,
-        screen_name.to_lowercase(),
-        screen_name,
-        current_account.username,
-        host,
-        app_data.client_id,
-        app_data.client_secret,
-        app_data.redirect_uri,
-        token_data.access_token,
-    );
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_get_attribute_from_url() {
-        let expected_credentials_field_attribute = CredentialsFieldAttribute {
-            name: "Nintendojo.fr".to_string(),
-            value: "https://www.nintendojo.fr".to_string(),
-        };
-
-        let true_urls = vec![UrlEntity {
-            display_url: "Nintendojo.fr".to_string(),
-            expanded_url: Some("https://www.nintendojo.fr".to_string()),
-            range: (1, 3),
-            url: "https://t.me/balek".to_string(),
-        }];
-
-        let false_urls = vec![UrlEntity {
-            display_url: "Nintendojo.fr".to_string(),
-            expanded_url: None,
-            range: (1, 3),
-            url: "https://t.me/balek".to_string(),
-        }];
-
-        assert!(get_attribute_from_url(&None).is_none());
-        assert!(get_attribute_from_url(&Some(UserEntityDetail { urls: false_urls })).is_none());
-
-        let binding = get_attribute_from_url(&Some(UserEntityDetail { urls: true_urls })).unwrap();
-        let result_credentials_field_attribute = binding.first().unwrap();
-
-        assert_eq!(
-            result_credentials_field_attribute.name,
-            expected_credentials_field_attribute.name
-        );
-        assert_eq!(
-            result_credentials_field_attribute.value,
-            expected_credentials_field_attribute.value
-        );
-    }
-
-    #[test]
-    fn test_get_note_from_description() {
-        let urls = vec![UrlEntity {
-            display_url: "tamerelol".to_string(),
-            expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
-            range: (1, 3),
-            url: "https://t.me/tamerelol".to_string(),
-        }];
-
-        let some_description = Some("Youpi | https://t.me/tamerelol".to_string());
-        let none_description = None;
-
-        assert_eq!(
-            get_note_from_description(&some_description, &urls),
-            Some("Youpi | https://www.nintendojo.fr/dojobar".to_string())
-        );
-
-        assert_eq!(get_note_from_description(&none_description, &urls), None);
-    }
-
-    #[test]
-    fn test_replace_tweet_by_toot() {
-        let mut associated_urls = HashMap::from([
-            (
-                "https://t.co/perdudeouf".to_string(),
-                "https://www.perdu.com".to_string(),
-            ),
-            (
-                "https://t.co/realquoteshere".to_string(),
-                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
-            ),
-            (
-                "https://t.co/almostthere".to_string(),
-                "https://twitter.com/NintendojoFR/status/nope".to_string(),
-            ),
-            (
-                "http://t.co/yetanotherone".to_string(),
-                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
-            ),
-        ]);
-
-        let expected_urls = HashMap::from([
-            (
-                "https://t.co/perdudeouf".to_string(),
-                "https://www.perdu.com".to_string(),
-            ),
-            (
-                "https://t.co/realquoteshere".to_string(),
-                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
-            ),
-            (
-                "https://t.co/almostthere".to_string(),
-                "https://twitter.com/NintendojoFR/status/nope".to_string(),
-            ),
-            (
-                "http://t.co/yetanotherone".to_string(),
-                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
-            ),
-        ]);
-
-        replace_tweet_by_toot(
-            &mut associated_urls,
-            "NintendojoFR",
-            1590047921633755136,
-            "nintendojofr",
-            "https://m.nintendojo.fr",
-            "109309605486908797",
-        );
-
-        assert_eq!(associated_urls, expected_urls);
-    }
-
-    #[test]
-    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
-        let masto_config = HashMap::from([
-            (
-                "test".to_string(),
-                MastodonConfig {
-                    twitter_screen_name: "tonpere".to_string(),
-                    mastodon_screen_name: Some("lalali".to_string()),
-                    twitter_page_size: None,
-                    base: "https://mstdn.net".to_string(),
-                    client_id: "".to_string(),
-                    client_secret: "".to_string(),
-                    redirect: "".to_string(),
-                    token: "".to_string(),
-                },
-            ),
-            (
-                "test2".to_string(),
-                MastodonConfig {
-                    twitter_screen_name: "tamerelol".to_string(),
-                    mastodon_screen_name: None,
-                    twitter_page_size: None,
-                    base: "https://mastoot.fr".to_string(),
-                    client_id: "".to_string(),
-                    client_secret: "".to_string(),
-                    redirect: "".to_string(),
-                    token: "".to_string(),
-                },
-            ),
-            (
-                "test3".to_string(),
-                MastodonConfig {
-                    twitter_screen_name: "NintendojoFR".to_string(),
-                    mastodon_screen_name: Some("nintendojofr".to_string()),
-                    twitter_page_size: None,
-                    base: "https://m.nintendojo.fr".to_string(),
-                    client_id: "".to_string(),
-                    client_secret: "".to_string(),
-                    redirect: "".to_string(),
-                    token: "".to_string(),
-                },
-            ),
-        ]);
-
-        // case sensitiveness, to avoid any mistake
-        assert_eq!(
-            None,
-            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
-        );
-        assert_eq!(
-            Some((
-                "nintendojofr".to_string(),
-                "https://m.nintendojo.fr".to_string()
-            )),
-            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
-        );
-        // should return None if twitter_screen_name is undefined
-        assert_eq!(
-            None,
-            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
-        );
-        assert_eq!(
-            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
-            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
-        );
-    }
-
-    #[test]
-    fn test_twitter_mentions() {
-        let mention_entities = vec![
-            MentionEntity {
-                id: 12345,
-                range: (1, 3),
-                name: "Ta Mere l0l".to_string(),
-                screen_name: "tamerelol".to_string(),
-            },
-            MentionEntity {
-                id: 6789,
-                range: (1, 3),
-                name: "TONPERE".to_string(),
-                screen_name: "tonpere".to_string(),
-            },
-        ];
-
-        let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
-
-        let masto_config = HashMap::from([(
-            "test".to_string(),
-            (MastodonConfig {
-                twitter_screen_name: "tonpere".to_string(),
-                mastodon_screen_name: Some("lalali".to_string()),
-                twitter_page_size: None,
-                base: "https://mstdn.net".to_string(),
-                client_id: "".to_string(),
-                client_secret: "".to_string(),
-                redirect: "".to_string(),
-                token: "".to_string(),
-            }),
-        )]);
-
-        twitter_mentions(&mut toot, &mention_entities, &masto_config);
-
-        assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
-    }
-
-    #[test]
-    fn test_decode_urls() {
-        let urls = HashMap::from([
-            (
-                "https://t.co/thisisatest".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
-            ),
-            (
-                "https://t.co/nopenotinclusive".to_string(),
-                "invité.es".to_string(),
-            ),
-        ]);
-
-        let mut toot =
-            "Rendez-vous sur https://t.co/thisisatest avec nos https://t.co/nopenotinclusive !"
-                .to_string();
-
-        decode_urls(&mut toot, &urls);
-
-        assert_eq!(
-            &toot,
-            "Rendez-vous sur https://www.nintendojo.fr/dojobar avec nos invité.es !"
-        );
-    }
-
-    #[test]
-    fn test_associate_urls() {
-        let urls = vec![
-            UrlEntity {
-                display_url: "tamerelol".to_string(),
-                expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
-                range: (1, 3),
-                url: "https://t.me/tamerelol".to_string(),
-            },
-            UrlEntity {
-                display_url: "sadcat".to_string(),
-                expanded_url: None,
-                range: (1, 3),
-                url: "https://t.me/sadcat".to_string(),
-            },
-            UrlEntity {
-                display_url: "invité.es".to_string(),
-                expanded_url: Some("http://xn--invit-fsa.es".to_string()),
-                range: (85, 108),
-                url: "https://t.co/WAUgnpHLmo".to_string(),
-            },
-        ];
-
-        let expected_urls = HashMap::from([
-            (
-                "https://t.me/tamerelol".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
-            ),
-            (
-                "https://t.co/WAUgnpHLmo".to_string(),
-                "invité.es".to_string(),
-            ),
-        ]);
-
-        let re = Regex::new("(.+)\\.es$").ok();
-
-        let associated_urls = associate_urls(&urls, &re);
-
-        assert_eq!(associated_urls, expected_urls);
-    }
-
-    #[test]
-    fn test_replace_alt_services() {
-        let mut associated_urls = HashMap::from([
-            (
-                "https://t.co/youplaboom".to_string(),
-                "https://www.youtube.com/watch?v=dQw4w9WgXcQ".to_string(),
-            ),
-            (
-                "https://t.co/thisisfine".to_string(),
-                "https://twitter.com/Nintendo/status/1594590628771688448".to_string(),
-            ),
-            (
-                "https://t.co/nopenope".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
-            ),
-            (
-                "https://t.co/broken".to_string(),
-                "http://youtu.be".to_string(),
-            ),
-            (
-                "https://t.co/alsobroken".to_string(),
-                "https://youtube.com".to_string(),
-            ),
-        ]);
-
-        let alt_services = HashMap::from([
-            ("twitter.com".to_string(), "nitter.net".to_string()),
-            ("youtu.be".to_string(), "invidio.us".to_string()),
-            ("www.youtube.com".to_string(), "invidio.us".to_string()),
-            ("youtube.com".to_string(), "invidio.us".to_string()),
-        ]);
-
-        let expected_urls = HashMap::from([
-            (
-                "https://t.co/youplaboom".to_string(),
-                "https://invidio.us/watch?v=dQw4w9WgXcQ".to_string(),
-            ),
-            (
-                "https://t.co/thisisfine".to_string(),
-                "https://nitter.net/Nintendo/status/1594590628771688448".to_string(),
-            ),
-            (
-                "https://t.co/nopenope".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
-            ),
-            (
-                "https://t.co/broken".to_string(),
-                "http://youtu.be".to_string(),
-            ),
-            (
-                "https://t.co/alsobroken".to_string(),
-                "https://youtube.com".to_string(),
-            ),
-        ]);
-
-        replace_alt_services(&mut associated_urls, &alt_services);
-
-        assert_eq!(associated_urls, expected_urls);
-    }
-}

src/state.rs

@@ -1,304 +0,0 @@
-use std::error::Error;
-
-use log::debug;
-
-use rusqlite::{params, Connection, OptionalExtension};
-
-/// Struct for each query line
-#[derive(Debug)]
-pub struct TweetToToot {
-    pub twitter_screen_name: String,
-    pub tweet_id: u64,
-    pub toot_id: String,
-}
-
-/// if None is passed, read the last tweet from DB
-/// if a tweet_id is passed, read this particular tweet from DB
-pub fn read_state(
-    conn: &Connection,
-    n: &str,
-    s: Option<u64>,
-) -> Result<Option<TweetToToot>, Box<dyn Error>> {
-    debug!("Reading tweet_id {:?}", s);
-    let query: String = match s {
-        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {} and twitter_screen_name = \"{}\"", i, n),
-        None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{}\" ORDER BY tweet_id DESC LIMIT 1", n),
-    };
-
-    let mut stmt = conn.prepare(&query)?;
-
-    let t = stmt
-        .query_row([], |row| {
-            Ok(TweetToToot {
-                twitter_screen_name: row.get("twitter_screen_name")?,
-                tweet_id: row.get("tweet_id")?,
-                toot_id: row.get("toot_id")?,
-            })
-        })
-        .optional()?;
-
-    Ok(t)
-}
-
-/// Writes last treated tweet id and toot id to the db
-pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
-    debug!("Write struct {:?}", t);
-    conn.execute(
-        "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id) VALUES (?1, ?2, ?3)",
-        params![t.twitter_screen_name, t.tweet_id, t.toot_id],
-    )?;
-
-    Ok(())
-}
-
-/// Initiates the DB from path
-pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
-    debug!("Initializing DB for Scootaloo");
-    let conn = Connection::open(d)?;
-
-    conn.execute(
-        "CREATE TABLE IF NOT EXISTS tweet_to_toot (
-            twitter_screen_name TEXT NOT NULL,
-            tweet_id            INTEGER PRIMARY KEY,
-            toot_id             TEXT UNIQUE
-            )",
-        [],
-    )?;
-
-    Ok(())
-}
-
-/// Migrate DB from 0.6.x to 0.7.x
-pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
-    debug!("Migrating DB for Scootaloo");
-
-    let conn = Connection::open(d)?;
-
-    let res = conn.execute(
-        &format!(
-            "ALTER TABLE tweet_to_toot
-        ADD COLUMN twitter_screen_name TEXT NOT NULL
-        DEFAULT \"{}\"",
-            s
-        ),
-        [],
-    );
-
-    match res {
-        Err(e) => match e.to_string().as_str() {
-            "duplicate column name: twitter_screen_name" => Ok(()),
-            _ => Err(e.into()),
-        },
-        _ => Ok(()),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::{fs::remove_file, path::Path};
-
-    #[test]
-    fn test_init_db() {
-        let d = "/tmp/test_init_db.sqlite";
-
-        init_db(d).unwrap();
-
-        // check that file exist
-        assert!(Path::new(d).exists());
-
-        // open said file
-        let conn = Connection::open(d).unwrap();
-        conn.execute("SELECT * from tweet_to_toot;", []).unwrap();
-
-        remove_file(d).unwrap();
-    }
-
-    #[test]
-    fn test_init_init_db() {
-        // init_db fn should be idempotent so let’s test that
-        let d = "/tmp/test_init_init_db.sqlite";
-
-        init_db(d).unwrap();
-
-        let conn = Connection::open(d).unwrap();
-
-        conn.execute(
-            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
-            VALUES
-            ('tamerelol', 100, 'A');",
-            [],
-        )
-        .unwrap();
-
-        init_db(d).unwrap();
-
-        remove_file(d).unwrap();
-    }
-
-    #[test]
-    fn test_write_state() {
-        let d = "/tmp/test_write_state.sqlite";
-
-        init_db(d).unwrap();
-
-        let conn = Connection::open(d).unwrap();
-
-        let t_in = TweetToToot {
-            twitter_screen_name: "tamerelol".to_string(),
-            tweet_id: 123456789,
-            toot_id: "987654321".to_string(),
-        };
-
-        write_state(&conn, t_in).unwrap();
-
-        let mut stmt = conn.prepare("SELECT * FROM tweet_to_toot;").unwrap();
-
-        let t_out = stmt
-            .query_row([], |row| {
-                Ok(TweetToToot {
-                    twitter_screen_name: row.get("twitter_screen_name").unwrap(),
-                    tweet_id: row.get("tweet_id").unwrap(),
-                    toot_id: row.get("toot_id").unwrap(),
-                })
-            })
-            .unwrap();
-
-        assert_eq!(&t_out.twitter_screen_name, "tamerelol");
-        assert_eq!(t_out.tweet_id, 123456789);
-        assert_eq!(&t_out.toot_id, "987654321");
-
-        remove_file(d).unwrap();
-    }
-
-    #[test]
-    fn test_none_to_tweet_id_read_state() {
-        let d = "/tmp/test_none_to_tweet_id_read_state.sqlite";
-
-        init_db(d).unwrap();
-
-        let conn = Connection::open(d).unwrap();
-
-        conn.execute(
-            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
-            VALUES
-            ('tamerelol', 101, 'A'),
-            ('tamerelol', 102, 'B');",
-            [],
-        )
-        .unwrap();
-
-        let t_out = read_state(&conn, "tamerelol", None).unwrap().unwrap();
-
-        remove_file(d).unwrap();
-
-        assert_eq!(t_out.tweet_id, 102);
-        assert_eq!(t_out.toot_id, "B");
-    }
-
-    #[test]
-    fn test_none_to_none_read_state() {
-        let d = "/tmp/test_none_to_none_read_state.sqlite";
-
-        init_db(d).unwrap();
-
-        let conn = Connection::open(d).unwrap();
-
-        let t_out = read_state(&conn, "tamerelol", None).unwrap();
-
-        remove_file(d).unwrap();
-
-        assert!(t_out.is_none());
-    }
-
-    #[test]
-    fn test_tweet_id_to_none_read_state() {
-        let d = "/tmp/test_tweet_id_to_none_read_state.sqlite";
-
-        init_db(d).unwrap();
-
-        let conn = Connection::open(d).unwrap();
-
-        conn.execute(
-            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
-            VALUES
-            ('tamerelol', 100, 'A');",
-            [],
-        )
-        .unwrap();
-
-        let t_out = read_state(&conn, "tamerelol", Some(101)).unwrap();
-
-        remove_file(d).unwrap();
-
-        assert!(t_out.is_none());
-    }
-
-    #[test]
-    fn test_tweet_id_to_tweet_id_read_state() {
-        let d = "/tmp/test_tweet_id_to_tweet_id_read_state.sqlite";
-
-        init_db(d).unwrap();
-
-        let conn = Connection::open(d).unwrap();
-
-        conn.execute(
-            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
-            VALUES
-            ('tamerelol', 100, 'A');",
-            [],
-        )
-        .unwrap();
-
-        let t_out = read_state(&conn, "tamerelol", Some(100)).unwrap().unwrap();
-
-        remove_file(d).unwrap();
-
-        assert_eq!(t_out.tweet_id, 100);
-        assert_eq!(t_out.toot_id, "A");
-    }
-
-    #[test]
-    fn test_migrate_db_add_column() {
-        let d = "/tmp/test_migrate_db_add_column.sqlite";
-
-        let conn = Connection::open(d).unwrap();
-
-        conn.execute(
-            "CREATE TABLE IF NOT EXISTS tweet_to_toot (
-            tweet_id            INTEGER PRIMARY KEY,
-            toot_id             TEXT UNIQUE
-            )",
-            [],
-        )
-        .unwrap();
-
-        migrate_db(d, "tamerelol").unwrap();
-
-        let mut stmt = conn.prepare("PRAGMA table_info(tweet_to_toot);").unwrap();
-
-        let mut t = stmt.query([]).unwrap();
-
-        while let Some(row) = t.next().unwrap() {
-            if row.get::<usize, u8>(0).unwrap() == 2 {
-                assert_eq!(
-                    row.get::<usize, String>(1).unwrap(),
-                    "twitter_screen_name".to_string()
-                );
-            }
-        }
-
-        remove_file(d).unwrap();
-    }
-
-    #[test]
-    fn test_migrate_db_no_add_column() {
-        let d = "/tmp/test_migrate_db_no_add_column.sqlite";
-
-        init_db(d).unwrap();
-
-        migrate_db(d, "tamerelol").unwrap();
-
-        remove_file(d).unwrap();
-    }
-}

src/twitter.rs

@@ -1,193 +0,0 @@
-use crate::config::TwitterConfig;
-use crate::util::cache_media;
-use crate::ScootalooError;
-
-use egg_mode::{
-    entities::{MediaEntity, MediaType},
-    tweet::{user_timeline, Tweet},
-    user::UserID,
-    KeyPair, Token,
-};
-use std::error::Error;
-
-/// Gets Twitter oauth2 token
-pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
-    let con_token = KeyPair::new(
-        config.consumer_key.to_owned(),
-        config.consumer_secret.to_owned(),
-    );
-    let access_token = KeyPair::new(
-        config.access_key.to_owned(),
-        config.access_secret.to_owned(),
-    );
-
-    Token::Access {
-        consumer: con_token,
-        access: access_token,
-    }
-}
-
-/// Gets Twitter user timeline, eliminate responses to others and reverse it
-pub async fn get_user_timeline(
-    screen_name: &str,
-    token: &Token,
-    lid: Option<u64>,
-    page_size: i32,
-) -> Result<Vec<Tweet>, Box<dyn Error>> {
-    // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
-        .with_page_size(page_size)
-        .older(lid)
-        .await?;
-
-    let mut feed: Vec<Tweet> = feed
-        .iter()
-        .cloned()
-        .filter(|t| match &t.in_reply_to_screen_name {
-            Some(r) => r.to_lowercase() == screen_name.to_lowercase(),
-            None => true,
-        })
-        .collect();
-
-    feed.reverse();
-
-    Ok(feed)
-}
-
-/// Retrieves a single media from a tweet and store it in a temporary file
-pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
-    match m.media_type {
-        MediaType::Photo => cache_media(&m.media_url_https, t).await,
-        _ => match &m.video_info {
-            Some(v) => match &v.variants.iter().find(|&x| x.content_type == "video/mp4") {
-                Some(u) => cache_media(&u.url, t).await,
-                None => Err(ScootalooError::new(&format!(
-                    "Media Type for {} is video but no mp4 file URL is available",
-                    &m.url
-                ))
-                .into()),
-            },
-            None => Err(ScootalooError::new(&format!(
-                "Media Type for {} is video but does not contain any video_info",
-                &m.url
-            ))
-            .into()),
-        },
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use egg_mode::entities::{
-        MediaSize, MediaSizes,
-        MediaType::{Gif, Photo},
-        ResizeMode::Crop,
-        ResizeMode::Fit,
-        VideoInfo, VideoVariant,
-    };
-    use std::fs::remove_dir_all;
-
-    const TMP_DIR: &'static str = "/tmp/scootaloo_get_tweet_media_test";
-
-    #[tokio::test]
-    async fn test_get_tweet_media() {
-        let m_photo = MediaEntity {
-            display_url: "pic.twitter.com/sHrwmP69Yv".to_string(),
-            expanded_url: "https://twitter.com/NintendojoFR/status/1555473821121056771/photo/1"
-                .to_string(),
-            id: 1555473771280080896,
-            range: (91, 114),
-            media_url: "http://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg".to_string(),
-            media_url_https: "https://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg"
-                .to_string(),
-            sizes: MediaSizes {
-                thumb: MediaSize {
-                    w: 150,
-                    h: 150,
-                    resize: Crop
-                },
-                small: MediaSize {
-                    w: 680,
-                    h: 510,
-                    resize: Fit
-                },
-                medium: MediaSize {
-                    w: 1200,
-                    h: 900,
-                    resize: Fit
-                },
-                large: MediaSize {
-                    w: 1280,
-                    h: 960,
-                    resize: Fit
-                }
-            },
-            source_status_id: None,
-            media_type: Photo,
-            url: "https://t.co/sHrwmP69Yv".to_string(),
-            video_info: None,
-            ext_alt_text: Some("Le menu «\u{a0}Classes » du jeu vidéo Xenoblade Chronicles 3 (Switch). L’affinité du personnage pour la classe est notée par quatre lettres : C, A, C, A (caca)."
-                .to_string())
-        };
-        let m_video = MediaEntity {
-            display_url: "pic.twitter.com/xDln0RrkjU".to_string(),
-            expanded_url: "https://twitter.com/NintendojoFR/status/1551822196833673218/photo/1"
-                .to_string(),
-            id: 1551822189711790081,
-            range: (275, 298),
-            media_url: "http://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg".to_string(),
-            media_url_https: "https://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg"
-                .to_string(),
-            sizes: MediaSizes {
-                thumb: MediaSize {
-                    w: 150,
-                    h: 150,
-                    resize: Crop,
-                },
-                small: MediaSize {
-                    w: 320,
-                    h: 240,
-                    resize: Fit,
-                },
-                medium: MediaSize {
-                    w: 320,
-                    h: 240,
-                    resize: Fit,
-                },
-                large: MediaSize {
-                    w: 320,
-                    h: 240,
-                    resize: Fit,
-                },
-            },
-            source_status_id: None,
-            media_type: Gif,
-            url: "https://t.co/xDln0RrkjU".to_string(),
-            video_info: Some(VideoInfo {
-                aspect_ratio: (4, 3),
-                duration_millis: None,
-                variants: vec![VideoVariant {
-                    bitrate: Some(0),
-                    content_type: "video/mp4".parse::<mime::Mime>().unwrap(),
-                    url: "https://video.twimg.com/tweet_video/FYkuD0RXEAE-iDx.mp4".to_string(),
-                }],
-            }),
-            ext_alt_text: Some("Scared Nintendo GIF".to_string()),
-        };
-
-        let tweet_media_photo = get_tweet_media(&m_photo, TMP_DIR).await.unwrap();
-        let tweet_media_video = get_tweet_media(&m_video, TMP_DIR).await.unwrap();
-
-        assert_eq!(
-            tweet_media_photo,
-            format!("{}/FZYnJ1qWIAAReHt.jpg", TMP_DIR)
-        );
-        assert_eq!(
-            tweet_media_video,
-            format!("{}/FYkuD0RXEAE-iDx.mp4", TMP_DIR)
-        );
-
-        remove_dir_all(TMP_DIR).unwrap();
-    }
-}

src/util.rs

@@ -1,167 +0,0 @@
-use crate::{twitter::get_tweet_media, ScootalooError};
-
-use base64::encode;
-use egg_mode::tweet::Tweet;
-use futures::{stream, stream::StreamExt};
-use log::{error, info, warn};
-use megalodon::{mastodon::Mastodon, megalodon::Megalodon};
-use reqwest::Url;
-use std::error::Error;
-use tokio::{
-    fs::{create_dir_all, remove_file, File},
-    io::copy,
-};
-
-/// Generate associative table between media ids and tweet extended entities
-pub async fn generate_media_ids(
-    tweet: &Tweet,
-    cache_path: &str,
-    mastodon: &Mastodon,
-) -> (String, Vec<String>) {
-    let mut media_url = "".to_string();
-    let mut media_ids: Vec<String> = vec![];
-
-    if let Some(m) = &tweet.extended_entities {
-        info!("{} medias in tweet", m.media.len());
-
-        let medias = m.media.clone();
-
-        let mut stream = stream::iter(medias)
-            .map(|media| {
-                // attribute media url
-                media_url = media.url.clone();
-
-                // clone everything we need
-                let cache_path = String::from(cache_path);
-                let mastodon = mastodon.clone();
-
-                tokio::task::spawn(async move {
-                    info!("Start treating {}", media.media_url_https);
-                    // get the tweet embedded media
-                    let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
-
-                    // upload media to Mastodon
-                    let mastodon_media = mastodon
-                        .upload_media(local_tweet_media_path.to_owned(), None)
-                        .await?
-                        .json();
-                    // at this point, we can safely erase the original file
-                    // it doesn’t matter if we can’t remove, cache_media fn is idempotent
-                    remove_file(&local_tweet_media_path).await.ok();
-
-                    Ok::<String, ScootalooError>(mastodon_media.id)
-                })
-            })
-            .buffered(4); // there are max four medias per tweet and they need to be treated in
-                          // order
-
-        while let Some(result) = stream.next().await {
-            match result {
-                Ok(Ok(v)) => media_ids.push(v),
-                Ok(Err(e)) => warn!("Cannot treat media: {}", e),
-                Err(e) => error!("Something went wrong when joining the main thread: {}", e),
-            }
-        }
-    } else {
-        info!("No media in tweet");
-    }
-
-    // in case some media_ids slot remained empty due to errors, remove them
-    media_ids.retain(|x| !x.is_empty());
-
-    (media_url, media_ids)
-}
-
-/// Transforms the media into a base64 equivalent
-pub async fn base64_media(u: &str) -> Result<String, Box<dyn Error>> {
-    let mut response = reqwest::get(u).await?;
-
-    let mut buffer = Vec::new();
-
-    while let Some(chunk) = response.chunk().await? {
-        copy(&mut &*chunk, &mut buffer).await?;
-    }
-
-    let content_type = response
-        .headers()
-        .get("content-type")
-        .ok_or_else(|| ScootalooError::new(&format!("Cannot get media content type for {}", u)))?
-        .to_str()?;
-
-    let encoded_f = encode(buffer);
-
-    Ok(format!("data:{};base64,{}", content_type, encoded_f))
-}
-
-/// Gets and caches Twitter Media inside the determined temp dir
-pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
-    // create dir
-    create_dir_all(t).await?;
-
-    // get file
-    let mut response = reqwest::get(u).await?;
-
-    // create local file
-    let url = Url::parse(u)?;
-    let dest_filename = url
-        .path_segments()
-        .ok_or_else(|| {
-            ScootalooError::new(&format!(
-                "Cannot determine the destination filename for {}",
-                u
-            ))
-        })?
-        .last()
-        .ok_or_else(|| {
-            ScootalooError::new(&format!(
-                "Cannot determine the destination filename for {}",
-                u
-            ))
-        })?;
-
-    let dest_filepath = format!("{}/{}", t, dest_filename);
-
-    let mut dest_file = File::create(&dest_filepath).await?;
-
-    while let Some(chunk) = response.chunk().await? {
-        copy(&mut &*chunk, &mut dest_file).await?;
-    }
-
-    Ok(dest_filepath)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use std::{fs::remove_dir_all, path::Path};
-
-    const TMP_DIR: &'static str = "/tmp/scootaloo_test";
-
-    #[tokio::test]
-    async fn test_cache_media() {
-        let dest = cache_media(
-            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
-            TMP_DIR,
-        )
-        .await
-        .unwrap();
-
-        assert!(Path::new(&dest).exists());
-
-        remove_dir_all(TMP_DIR).unwrap();
-    }
-
-    #[tokio::test]
-    async fn test_base64_media() {
-        let img = base64_media(
-            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
-        )
-        .await
-        .unwrap();
-
-        assert!(img.starts_with("data:image/png;base64,"));
-
-        assert!(img.ends_with("="));
-    }
-}

tests/bad_test.toml

@@ -1 +0,0 @@
-blah

tests/config.rs

@@ -1,171 +0,0 @@
-use scootaloo::parse_toml;
-use std::collections::HashMap;
-
-#[test]
-fn test_alt_services() {
-    let toml = parse_toml("tests/no_test_alt_services.toml");
-    assert_eq!(toml.scootaloo.alternative_services_for, None);
-
-    let toml = parse_toml("tests/test_alt_services.toml");
-    assert_eq!(
-        toml.scootaloo.alternative_services_for,
-        Some(HashMap::from([
-            ("tamere.lol".to_string(), "tonpere.mdr".to_string()),
-            ("you.pi".to_string(), "you.pla".to_string())
-        ]))
-    );
-}
-
-#[test]
-fn test_re_display() {
-    let toml = parse_toml("tests/no_show_url_as_display_url_for.toml");
-    assert_eq!(toml.scootaloo.show_url_as_display_url_for, None);
-
-    let toml = parse_toml("tests/show_url_as_display_url_for.toml");
-
-    assert_eq!(
-        toml.scootaloo.show_url_as_display_url_for,
-        Some("^(.+)\\.es$".to_string())
-    );
-}
-
-#[test]
-fn test_page_size() {
-    const DEFAULT_PAGE_SIZE: i32 = 200;
-    let toml = parse_toml("tests/page_size.toml");
-
-    assert_eq!(toml.twitter.page_size, Some(100));
-
-    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
-
-    assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42));
-
-    // this is the exact line that is used inside fn run() to determine the twitter page size
-    // passed to fn get_user_timeline()
-    let page_size_for_0 = toml
-        .mastodon
-        .get("0")
-        .unwrap()
-        .twitter_page_size
-        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
-    let page_size_for_1 = toml
-        .mastodon
-        .get("1")
-        .unwrap()
-        .twitter_page_size
-        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
-
-    assert_eq!(page_size_for_0, 100);
-    assert_eq!(page_size_for_1, 42);
-
-    let toml = parse_toml("tests/no_page_size.toml");
-
-    assert_eq!(toml.twitter.page_size, None);
-    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
-
-    // and same here
-    let page_size_for_0 = toml
-        .mastodon
-        .get("0")
-        .unwrap()
-        .twitter_page_size
-        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
-
-    assert_eq!(page_size_for_0, 200);
-}
-
-#[test]
-fn test_parse_good_toml_rate_limit() {
-    let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml");
-
-    assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize));
-}
-
-#[test]
-fn test_parse_good_toml_mastodon_screen_name() {
-    let parse_good_toml = parse_toml("tests/good_test_mastodon_screen_name.toml");
-
-    assert_eq!(
-        parse_good_toml
-            .mastodon
-            .get("0")
-            .unwrap()
-            .mastodon_screen_name,
-        Some("tarace".to_string())
-    );
-
-    assert_eq!(
-        parse_good_toml
-            .mastodon
-            .get("1")
-            .unwrap()
-            .mastodon_screen_name,
-        None
-    );
-}
-
-#[test]
-fn test_parse_good_toml() {
-    let parse_good_toml = parse_toml("tests/good_test.toml");
-
-    assert_eq!(
-        parse_good_toml.scootaloo.db_path,
-        "/var/random/scootaloo.sqlite"
-    );
-    assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
-    assert_eq!(parse_good_toml.scootaloo.rate_limit, None);
-
-    assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
-    assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");
-    assert_eq!(parse_good_toml.twitter.access_key, "rand access key");
-    assert_eq!(parse_good_toml.twitter.access_secret, "super secret");
-
-    assert_eq!(
-        &parse_good_toml
-            .mastodon
-            .get("tamerelol")
-            .unwrap()
-            .twitter_screen_name,
-        "tamerelol"
-    );
-    assert_eq!(
-        &parse_good_toml.mastodon.get("tamerelol").unwrap().base,
-        "https://m.nintendojo.fr"
-    );
-    assert_eq!(
-        &parse_good_toml.mastodon.get("tamerelol").unwrap().client_id,
-        "rand client id"
-    );
-    assert_eq!(
-        &parse_good_toml
-            .mastodon
-            .get("tamerelol")
-            .unwrap()
-            .client_secret,
-        "secret"
-    );
-    assert_eq!(
-        &parse_good_toml.mastodon.get("tamerelol").unwrap().redirect,
-        "urn:ietf:wg:oauth:2.0:oob"
-    );
-    assert_eq!(
-        &parse_good_toml.mastodon.get("tamerelol").unwrap().token,
-        "super secret"
-    );
-}
-
-#[test]
-#[should_panic(
-    expected = "Cannot open config file tests/no_file.toml: No such file or directory (os error 2)"
-)]
-fn test_parse_no_toml() {
-    let _parse_no_toml = parse_toml("tests/no_file.toml");
-}
-
-#[test]
-#[should_panic(
-    expected = "Cannot parse TOML file tests/bad_test.toml: expected an equals, found a newline at line 1 column 5"
-)]
-fn test_parse_bad_toml() {
-    let _parse_bad_toml = parse_toml("tests/bad_test.toml");
-}

tests/good_test.toml

@@ -1,19 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.tamerelol]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/good_test_mastodon_screen_name.toml

@@ -1,28 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.0]
-twitter_screen_name="tamerelol"
-mastodon_screen_name="tarace"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"
-
-[mastodon.1]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/good_test_rate_limit.toml

@@ -1,20 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-rate_limit=69
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.tamerelol]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/no_page_size.toml

@@ -1,19 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.0]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/no_show_url_as_display_url_for.toml

@@ -1,19 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.tamerelol]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/no_test_alt_services.toml

@@ -1,19 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.tamerelol]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/page_size.toml

@@ -1,29 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-page_size=100
-
-[mastodon]
-[mastodon.0]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"
-
-[mastodon.1]
-twitter_screen_name="tonperemdr"
-twitter_page_size=42
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/show_url_as_display_url_for.toml

@@ -1,20 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-show_url_as_display_url_for = "^(.+)\\.es$"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.tamerelol]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"

tests/test_alt_services.toml

@@ -1,22 +0,0 @@
-[scootaloo]
-
-db_path="/var/random/scootaloo.sqlite"
-cache_path="/tmp/scootaloo"
-[scootaloo.alternative_services_for]
-"tamere.lol" = "tonpere.mdr"
-"you.pi" = "you.pla"
-
-[twitter]
-consumer_key="rand consumer key"
-consumer_secret="secret"
-access_key="rand access key"
-access_secret="super secret"
-
-[mastodon]
-[mastodon.tamerelol]
-twitter_screen_name="tamerelol"
-base = "https://m.nintendojo.fr"
-client_id = "rand client id"
-client_secret = "secret"
-redirect = "urn:ietf:wg:oauth:2.0:oob"
-token = "super secret"