Update README.md

Update file README.md

See merge request veretcle/scootaloo!57

Cargo.toml

@@ -1,25 +1,29 @@
 [package]
 name = "scootaloo"
-version = "0.6.0"
+version = "1.1.6"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-chrono = "^0.4"
+base64 = "^0.13"
+regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^2.34"
-futures = "^0.3"
+clap = "^4"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-tokio = { version = "1", features = ["full"]}
-elefren = "^0.22"
+tokio = { version = "^1", features = ["rt"]}
+futures = "^0.3"
+megalodon = "^0.3.6"
 html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
 simple_logger = "^2.1"
+mime = "^0.3"
 
 [profile.release]
 strip = true
+lto = true
+codegen-units = 1

README.md

@@ -1,3 +1,9 @@
+**Due to the new Twitter policy about API v1.1 and API v2, this project will no longer be updated as it no longer can work for free any way, shape or form.**
+
+First of all, I’m deeply sorry about that. It worked pretty great for what it did with a level of quality I’m very proud to have achieved.
+
+Secondly, fuck you Musk, fuck you.
+
 A Twitter to Mastodon copy bot written in Rust
 
 It:
@@ -10,23 +16,38 @@ If any of the last steps failed, the Toot gets published with the exact same tex
 RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
 
 # Usage
+## Configuring
 
 First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this:
 
 ```toml
 [scootaloo]
-
-db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
-cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
+cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneously
+## optional, this should be omitted the majority of the time
+## sometimes, twitter try to use french inclusive writting, but instead of using `·` (median point), they’re using `.`
+## this makes twitter interpret it as a URL, which is wrong
+## this parameter allows you to catch such URLs and apply the `display_url` (i.e. `tout.es`) instead of the `expanded_url` (i.e. `http://tout.es`)
+## in those particular cases
+## (!) use with caution, it might have some undesired effects
+show_url_as_display_url_for = "^https?://(.+)\\.es$"
+## optional, this allows you to replace the host for popular services such as YouTube of Twitter, or any other
+## with their more freely accessible equivalent
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr" ## quotes are necessary for both parameters
+"you.pi" = "you.pla"
+"www.you.pi" = "you.pla" ## this is an exact match, so you’ll need to lay out all the possibilities
 
 [twitter]
-username="NintendojoFR" ## User Timeline to copy
-
 ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
-consumer_key="MYCONSUMERKEY"
-consumer_secret="MYCONSUMERSECRET"
-access_key="MYACCESSKEY"
-access_secret="MYACCESSSECRET"
+page_size = 20 ## optional, default 200, max number of tweet retrieved
+consumer_key = "MYCONSUMERKEY"
+consumer_secret = "MYCONSUMERSECRET"
+access_key = "MYACCESSKEY"
+access_secret = "MYACCESSSECRET"
+
+[mastodon]
 ```
 
 Then run the command with the `init` subcommand to initiate the DB:
@@ -44,7 +65,9 @@ scootaloo register --host https://m.nintendojo.fr
 This will give you the end of the TOML file. It will look like this:
 
 ```toml
-[mastodon]
+[mastodon.nintendojofr] ## account
+twitter_screen_name = "NintendojoFR" ## User Timeline to copy
+mastodon_screen_name = "nintendojofr" ## optional, Mastodon account name used for smart mentions
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
 client_secret = "MYCLIENTSECRET"
@@ -52,26 +75,34 @@ redirect = "urn:ietf:wg:oauth:2.0:oob"
 token = "MYTOKEN"
 ```
 
+You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
+
+You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so:
+```
+twitter_page_size = 40
+```
+
+## Running
+
 You can then run the application via `cron` for example. Here is the generic usage:
 
 ```sh
 A Twitter to Mastodon bot
 
-USAGE:
-    scootaloo [OPTIONS] [SUBCOMMAND]
+Usage: scootaloo [OPTIONS] [COMMAND]
 
-FLAGS:
-    -h, --help       Prints help information
-    -V, --version    Prints version information
+Commands:
+  register     Command to register to a Mastodon Instance
+  init         Command to init Scootaloo DB
+  migrate      Command to migrate Scootaloo DB
+  copyprofile  Command to copy a Twitter profile into Mastodon
+  help         Print this message or the help of the given subcommand(s)
 
-OPTIONS:
-    -c, --config <CONFIG_FILE>    TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)
-    -l, --loglevel <LOGLEVEL>     Log level. Valid values are: Off, Warn, Error, Info, Debug
-
-SUBCOMMANDS:
-    help        Prints this message or the help of the given subcommand(s)
-    init        Command to init Scootaloo DB
-    register    Command to register to a Mastodon Instance
+Options:
+  -c, --config <CONFIG_FILE>  TOML config file for scootaloo [default: /usr/local/etc/scootaloo.toml]
+  -l, --loglevel <LOGLEVEL>   Log level [possible values: Off, Warn, Error, Info, Debug]
+  -h, --help                  Print help information
+  -V, --version               Print version information
 ```
 
 # Quirks
@@ -86,5 +117,17 @@ sqlite3 /var/lib/scootaloo/scootaloo.sqlite
 And inserting the data:
 
 ```sql
-INSERT INTO tweet_to_toot VALUES (1383782580412030982, "");
+INSERT INTO tweet_to_toot VALUES ("<twitter_screen_name>", 1383782580412030982, "<twitter_screen_name>");
 ```
+
+The last value is supposed to be the Toot ID. It cannot be null, so you better initialize it with something unique, like the Twitter Screen Name for example.
+
+# Migrating from Scootaloo ⩽ 0.6.1
+
+The DB scheme has change between version 0.6.x and 0.7.x (this is due to the multi-account nature of Scootaloo from 0.7.x onward). You need to migrate your DB. You can do so by issuing the command:
+
+```
+scootaloo migrate
+```
+
+You can optionnally specify a screen name with the `--name` option. By default, it’ll take the first screen name in the config file.

src/config.rs

@@ -1,25 +1,29 @@
+use std::{collections::HashMap, fs::read_to_string};
+
 use serde::Deserialize;
-use std::fs::read_to_string;
 
 /// General configuration Struct
 #[derive(Debug, Deserialize)]
 pub struct Config {
     pub twitter: TwitterConfig,
-    pub mastodon: MastodonConfig,
+    pub mastodon: HashMap<String, MastodonConfig>,
     pub scootaloo: ScootalooConfig,
 }
 
 #[derive(Debug, Deserialize)]
 pub struct TwitterConfig {
-    pub username: String,
     pub consumer_key: String,
     pub consumer_secret: String,
     pub access_key: String,
     pub access_secret: String,
+    pub page_size: Option<i32>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
 pub struct MastodonConfig {
+    pub twitter_screen_name: String,
+    pub mastodon_screen_name: Option<String>,
+    pub twitter_page_size: Option<i32>,
     pub base: String,
     pub client_id: String,
     pub client_secret: String,
@@ -31,15 +35,18 @@ pub struct MastodonConfig {
 pub struct ScootalooConfig {
     pub db_path: String,
     pub cache_path: String,
+    pub rate_limit: Option<usize>,
+    pub show_url_as_display_url_for: Option<String>,
+    pub alternative_services_for: Option<HashMap<String, String>>,
 }
 
 /// Parses the TOML file into a Config Struct
 pub fn parse_toml(toml_file: &str) -> Config {
     let toml_config = read_to_string(toml_file)
-        .unwrap_or_else(|e| panic!("Cannot open config file {}: {}", toml_file, e));
+        .unwrap_or_else(|e| panic!("Cannot open config file {toml_file}: {e}"));
 
     let config: Config = toml::from_str(&toml_config)
-        .unwrap_or_else(|e| panic!("Cannot parse TOML file {}: {}", toml_file, e));
+        .unwrap_or_else(|e| panic!("Cannot parse TOML file {toml_file}: {e}"));
 
     config
 }

src/error.rs

@@ -1,8 +1,12 @@
 use std::{
+    boxed::Box,
+    convert::From,
     error::Error,
     fmt::{Display, Formatter, Result},
 };
 
+use megalodon::error::Error as megalodonError;
+
 #[derive(Debug)]
 pub struct ScootalooError {
     details: String,
@@ -23,3 +27,15 @@ impl Display for ScootalooError {
         write!(f, "{}", self.details)
     }
 }
+
+impl From<Box<dyn Error>> for ScootalooError {
+    fn from(error: Box<dyn Error>) -> Self {
+        ScootalooError::new(&format!("Error in a subset crate: {error}"))
+    }
+}
+
+impl From<megalodonError> for ScootalooError {
+    fn from(error: megalodonError) -> Self {
+        ScootalooError::new(&format!("Error in megalodon crate: {error}"))
+    }
+}

src/lib.rs

@@ -7,151 +7,306 @@ use config::Config;
 
 mod mastodon;
 pub use mastodon::register;
-use mastodon::{build_basic_status, get_mastodon_token};
+use mastodon::*;
 
 mod twitter;
 use twitter::*;
 
 mod util;
+use util::{base64_media, generate_media_ids};
 
 mod state;
-pub use state::init_db;
+pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
-use elefren::{prelude::*, status_builder::StatusBuilder};
-use log::{debug, error, info, warn};
+use futures::StreamExt;
+use html_escape::decode_html_entities;
+use log::info;
+use megalodon::{
+    megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
+};
+use regex::Regex;
 use rusqlite::Connection;
-use std::borrow::Cow;
-use tokio::fs::remove_file;
+use std::sync::Arc;
+use tokio::{spawn, sync::Mutex};
+
+const DEFAULT_RATE_LIMIT: usize = 4;
+const DEFAULT_PAGE_SIZE: i32 = 200;
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
     // open the SQLite connection
-    let conn = Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
-        panic!(
-            "Something went wrong when opening the DB {}: {}",
-            &config.scootaloo.db_path, e
-        )
-    });
-    // retrieve the last tweet ID for the username
-    let last_tweet_id = read_state(&conn, None)
-        .unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
-        .map(|s| s.tweet_id);
-
-    // get OAuth2 token
-    let token = get_oauth2_token(&config.twitter);
-
-    // get Mastodon instance
-    let mastodon = get_mastodon_token(&config.mastodon);
-
-    // get user timeline feed (Vec<tweet>)
-    let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
-        .await
-        .unwrap_or_else(|e| {
+    let conn = Arc::new(Mutex::new(
+        Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
             panic!(
-                "Something went wrong when trying to retrieve {}’s timeline: {}",
-                &config.twitter.username, e
+                "Something went wrong when opening the DB {}: {}",
+                &config.scootaloo.db_path, e
             )
-        });
+        }),
+    ));
 
-    // empty feed -> exiting
-    if feed.is_empty() {
-        info!("Nothing to retrieve since last time, exiting…");
-        return;
-    }
+    let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
 
-    // order needs to be chronological
-    feed.reverse();
+    let display_url_re = config
+        .scootaloo
+        .show_url_as_display_url_for
+        .as_ref()
+        .map(|r|
+        // we want to panic in case the RE is not valid
+        Regex::new(r).unwrap());
 
-    for tweet in &feed {
-        debug!("Treating Tweet {} inside feed", tweet.id);
-        // initiate the toot_reply_id var
-        let mut toot_reply_id: Option<String> = None;
-        // determine if the tweet is part of a thread (response to self) or a standard response
-        if let Some(r) = &tweet.in_reply_to_screen_name {
-            if r.to_lowercase() != config.twitter.username.to_lowercase() {
-                // we are responding not threading
-                info!("Tweet is a direct response, skipping");
-                continue;
-            }
-            info!("Tweet is a thread");
-            toot_reply_id = read_state(&conn, tweet.in_reply_to_status_id)
-                .unwrap_or(None)
-                .map(|s| s.toot_id);
-        };
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            // calculate Twitter page size
+            let page_size = mastodon_config
+                .twitter_page_size
+                .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
 
-        // build basic status by just yielding text and dereferencing contained urls
-        let mut status_text = build_basic_status(tweet);
+            // create temporary value for each task
+            let scootaloo_cache_path = config.scootaloo.cache_path.clone();
+            let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
+            let display_url_re = display_url_re.clone();
+            let token = get_oauth2_token(&config.twitter);
+            let task_conn = conn.clone();
+            let global_mastodon_config = global_mastodon_config.clone();
 
-        let mut status_medias: Vec<String> = vec![];
-        // reupload the attachments if any
-        if let Some(m) = &tweet.extended_entities {
-            for media in &m.media {
-                let local_tweet_media_path =
-                    match get_tweet_media(media, &config.scootaloo.cache_path).await {
-                        Ok(m) => m,
-                        Err(e) => {
-                            error!("Cannot get tweet media for {}: {}", &media.url, e);
-                            continue;
+            spawn(async move {
+                info!("Starting treating {}", &mastodon_config.twitter_screen_name);
+                // retrieve the last tweet ID for the username
+                let lconn = task_conn.lock().await;
+                let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
+                    .map(|r| r.tweet_id);
+                drop(lconn);
+
+                // get reversed, curated user timeline
+                let feed = get_user_timeline(
+                    &mastodon_config.twitter_screen_name,
+                    &token,
+                    last_tweet_id,
+                    page_size,
+                )
+                .await?;
+
+                // get Mastodon instance
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                for tweet in &feed {
+                    info!("Treating Tweet {} inside feed", tweet.id);
+
+                    // basic toot text
+                    let mut status_text = tweet.text.clone();
+
+                    // add mentions and smart mentions
+                    if !&tweet.entities.user_mentions.is_empty() {
+                        info!("Tweet contains mentions, add them!");
+                        let global_mastodon_config = global_mastodon_config.lock().await;
+                        twitter_mentions(
+                            &mut status_text,
+                            &tweet.entities.user_mentions,
+                            &global_mastodon_config,
+                        );
+                        drop(global_mastodon_config);
+                    }
+
+                    if !&tweet.entities.urls.is_empty() {
+                        info!("Tweet contains links, add them!");
+                        let mut associated_urls =
+                            associate_urls(&tweet.entities.urls, &display_url_re);
+
+                        if let Some(q) = &tweet.quoted_status {
+                            if let Some(u) = &q.user {
+                                info!(
+                                    "Tweet {} contains a quote, we try to find it within the DB",
+                                    tweet.id
+                                );
+                                // we know we have a quote and a user, we can lock both the
+                                // connection to DB and global_config
+                                // we will release them manually as soon as they’re useless
+                                let lconn = task_conn.lock().await;
+                                let global_mastodon_config = global_mastodon_config.lock().await;
+                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
+                                {
+                                    info!("We have found the associated toot({})", &r.toot_id);
+                                    // drop conn immediately after the request: we won’t need it
+                                    // any more and the treatment there might be time-consuming
+                                    drop(lconn);
+                                    if let Some((m, t)) =
+                                        find_mastodon_screen_name_by_twitter_screen_name(
+                                            &r.twitter_screen_name,
+                                            &global_mastodon_config,
+                                        )
+                                    {
+                                        // drop the global conf, we have all we required, no need
+                                        // to block it further
+                                        drop(global_mastodon_config);
+                                        replace_tweet_by_toot(
+                                            &mut associated_urls,
+                                            &r.twitter_screen_name,
+                                            q.id,
+                                            &m,
+                                            &t,
+                                            &r.toot_id,
+                                        );
+                                    }
+                                }
+                            }
                         }
+
+                        if let Some(a) = &scootaloo_alt_services {
+                            replace_alt_services(&mut associated_urls, a);
+                        }
+
+                        decode_urls(&mut status_text, &associated_urls);
+                    }
+
+                    // building associative media list
+                    let (media_url, status_medias) =
+                        generate_media_ids(tweet, &scootaloo_cache_path, &mastodon).await;
+
+                    status_text = status_text.replace(&media_url, "");
+
+                    // now that the text won’t be altered anymore, we can safely remove HTML
+                    // entities
+                    status_text = decode_html_entities(&status_text).to_string();
+
+                    info!("Building corresponding Mastodon status");
+
+                    let mut post_status = PostStatusInputOptions {
+                        media_ids: None,
+                        poll: None,
+                        in_reply_to_id: None,
+                        sensitive: None,
+                        spoiler_text: None,
+                        visibility: None,
+                        scheduled_at: None,
+                        language: None,
+                        quote_id: None,
                     };
 
-                let mastodon_media_ids = match mastodon
-                    .media(Cow::from(local_tweet_media_path.to_owned()))
-                {
-                    Ok(m) => {
-                        remove_file(&local_tweet_media_path)
-                            .await
-                            .unwrap_or_else(|e|
-                                warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
-                            );
-                        m.id
+                    if !status_medias.is_empty() {
+                        post_status.media_ids = Some(status_medias);
                     }
-                    Err(e) => {
-                        error!(
-                            "Attachment {} cannot be uploaded to Mastodon Instance: {}",
-                            &local_tweet_media_path, e
-                        );
-                        continue;
+
+                    // thread if necessary
+                    if tweet.in_reply_to_user_id.is_some() {
+                        let lconn = task_conn.lock().await;
+                        if let Ok(Some(r)) = read_state(
+                            &lconn,
+                            &mastodon_config.twitter_screen_name,
+                            tweet.in_reply_to_status_id,
+                        ) {
+                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
+                        }
+                        drop(lconn);
                     }
-                };
 
-                status_medias.push(mastodon_media_ids);
+                    // language if any
+                    if let Some(l) = &tweet.lang {
+                        post_status.language = Some(l.to_string());
+                    }
 
-                // last step, removing the reference to the media from with the toot’s text
-                status_text = status_text.replace(&media.url, "");
-            }
+                    // can be activated for test purposes
+                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
+
+                    let published_status = mastodon
+                        .post_status(status_text, Some(&post_status))
+                        .await?
+                        .json();
+                    // this will return if it cannot publish the status preventing the last_tweet from
+                    // being written into db
+
+                    let ttt_towrite = TweetToToot {
+                        twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
+                        tweet_id: tweet.id,
+                        toot_id: published_status.id,
+                    };
+
+                    // write the current state (tweet ID and toot ID) to avoid copying it another time
+                    let lconn = task_conn.lock().await;
+                    write_state(&lconn, ttt_towrite)?;
+                    drop(lconn);
+                }
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    // launch and wait for every handle
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {e}"),
+            Err(e) => eprintln!("Error with thread: {e}"),
+            _ => (),
+        }
+    }
+}
+
+/// Copies the Twitter profile into Mastodon
+#[tokio::main]
+pub async fn profile(config: Config, bot: Option<bool>) {
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            let token = get_oauth2_token(&config.twitter);
+
+            spawn(async move {
+                // get the user of the last tweet of the feed
+                let twitter_user =
+                    get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
+                        .await?
+                        .first()
+                        .ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
+                        .clone()
+                        .user
+                        .ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
+
+                let note = get_note_from_description(
+                    &twitter_user.description,
+                    &twitter_user.entities.description.urls,
+                );
+
+                let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
+
+                let display_name = Some(String::from_utf16_lossy(
+                    &twitter_user
+                        .name
+                        .encode_utf16()
+                        .take(30)
+                        .collect::<Vec<u16>>(),
+                ));
+
+                let header = match twitter_user.profile_banner_url {
+                    Some(h) => Some(base64_media(&h).await?),
+                    None => None,
+                };
+
+                let update_creds = UpdateCredentialsInputOptions {
+                    bot,
+                    display_name,
+                    note,
+                    avatar: Some(
+                        base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
+                            .await?,
+                    ),
+                    header,
+                    fields_attributes,
+                    ..Default::default()
+                };
+
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                mastodon.update_credentials(Some(&update_creds)).await?;
+
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {e}"),
+            Err(e) => eprintln!("Error with thread: {e}"),
+            _ => (),
         }
-        // finished reuploading attachments, now let’s do the toot baby!
-
-        debug!("Building corresponding Mastodon status");
-
-        let mut status_builder = StatusBuilder::new();
-
-        status_builder.status(&status_text).media_ids(status_medias);
-
-        if let Some(i) = toot_reply_id {
-            status_builder.in_reply_to(&i);
-        }
-
-        let status = status_builder
-            .build()
-            .unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
-
-        // publish status
-        // again unwrap is safe here as we are in the main thread
-        let published_status = mastodon.new_status(status).unwrap();
-        // this will panic if it cannot publish the status, which is a good thing, it allows the
-        // last_tweet gathered not to be written
-
-        let ttt_towrite = TweetToToot {
-            tweet_id: tweet.id,
-            toot_id: published_status.id,
-        };
-
-        // write the current state (tweet ID and toot ID) to avoid copying it another time
-        write_state(&conn, ttt_towrite)
-            .unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e));
     }
 }

src/main.rs

@@ -1,5 +1,5 @@
-use clap::{App, Arg, SubCommand};
-use log::{error, LevelFilter};
+use clap::{Arg, ArgAction, Command};
+use log::LevelFilter;
 use scootaloo::*;
 use simple_logger::SimpleLogger;
 use std::str::FromStr;
@@ -7,91 +7,177 @@ use std::str::FromStr;
 const DEFAULT_CONFIG_PATH: &str = "/usr/local/etc/scootaloo.toml";
 
 fn main() {
-    let matches = App::new(env!("CARGO_PKG_NAME"))
+    let matches = Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))
         .about("A Twitter to Mastodon bot")
         .arg(
-            Arg::with_name("config")
-                .short("c")
+            Arg::new("config")
+                .short('c')
                 .long("config")
                 .value_name("CONFIG_FILE")
-                .help(&format!(
-                    "TOML config file for scootaloo (default {})",
-                    DEFAULT_CONFIG_PATH
-                ))
-                .takes_value(true)
+                .help("TOML config file for scootaloo")
+                .num_args(1)
+                .default_value(DEFAULT_CONFIG_PATH)
                 .display_order(1),
         )
         .arg(
-            Arg::with_name("log_level")
-                .short("l")
+            Arg::new("log_level")
+                .short('l')
                 .long("loglevel")
                 .value_name("LOGLEVEL")
-                .help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
-                .takes_value(true)
+                .help("Log level")
+                .num_args(1)
+                .value_parser(["Off", "Warn", "Error", "Info", "Debug"])
                 .display_order(2),
         )
         .subcommand(
-            SubCommand::with_name("register")
+            Command::new("register")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to register to a Mastodon Instance")
                 .arg(
-                    Arg::with_name("host")
-                        .short("H")
+                    Arg::new("host")
+                        .short('H')
                         .long("host")
                         .value_name("HOST")
                         .help("Base URL of the Mastodon instance to register to (no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .required(true)
+                        .display_order(1)
+                    )
+                .arg(
+                    Arg::new("name")
+                        .short('n')
+                        .long("name")
+                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                        .num_args(1)
+                        .required(true)
+                        .display_order(2)
+                ),
+        )
+        .subcommand(
+            Command::new("init")
+                .version(env!("CARGO_PKG_VERSION"))
+                .about("Command to init Scootaloo DB")
+                .arg(
+                    Arg::new("config")
+                        .short('c')
+                        .long("config")
+                        .value_name("CONFIG_FILE")
+                        .help("TOML config file for scootaloo")
+                        .default_value(DEFAULT_CONFIG_PATH)
+                        .num_args(1)
                         .display_order(1),
                 ),
         )
         .subcommand(
-            SubCommand::with_name("init")
-                .version(env!("CARGO_PKG_VERSION"))
-                .about("Command to init Scootaloo DB")
-                .arg(
-                    Arg::with_name("config")
-                        .short("c")
-                        .long("config")
-                        .value_name("CONFIG_FILE")
-                        .help(&format!(
-                            "TOML config file for scootaloo (default {})",
-                            DEFAULT_CONFIG_PATH
-                        ))
-                        .takes_value(true)
-                        .display_order(1),
-                ),
+            Command::new("migrate")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to migrate Scootaloo DB")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help("TOML config file for scootaloo")
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+        )
+        .subcommand(
+            Command::new("copyprofile")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to copy a Twitter profile into Mastodon")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help("TOML config file for scootaloo")
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Mastodon Config name (as seen in the config file)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+            .arg(
+                Arg::new("bot")
+                    .short('b')
+                    .long("bot")
+                    .help("Declare user as bot")
+                    .action(ArgAction::SetTrue)
+                    .display_order(3)
+            )
         )
         .get_matches();
 
     match matches.subcommand() {
-        ("register", Some(sub_m)) => {
-            register(sub_m.value_of("host").unwrap());
+        Some(("register", sub_m)) => {
+            register(
+                sub_m.get_one::<String>("host").unwrap(),
+                sub_m.get_one::<String>("name").unwrap(),
+            );
             return;
         }
-        ("init", Some(sub_m)) => {
-            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+        Some(("init", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
             init_db(&config.scootaloo.db_path).unwrap();
             return;
         }
+        Some(("migrate", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            let config_twitter_screen_name =
+                &config.mastodon.values().next().unwrap().twitter_screen_name;
+            migrate_db(
+                &config.scootaloo.db_path,
+                sub_m
+                    .get_one::<String>("name")
+                    .unwrap_or(config_twitter_screen_name),
+            )
+            .unwrap();
+            return;
+        }
+        Some(("copyprofile", sub_m)) => {
+            let mut config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            // filters out the user passed in cli from the global configuration
+            if let Some(m) = sub_m.get_one::<String>("name") {
+                match config.mastodon.get(m) {
+                    Some(_) => {
+                        config.mastodon.retain(|k, _| k == m);
+                    }
+                    None => panic!("Config file does not contain conf for {}", &m),
+                }
+            }
+
+            profile(config, sub_m.get_flag("bot").then_some(true));
+
+            return;
+        }
         _ => (),
     }
 
-    if matches.is_present("log_level") {
-        match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
-            Ok(level) => SimpleLogger::new().with_level(level).init().unwrap(),
-            Err(e) => {
-                SimpleLogger::new()
-                    .with_level(LevelFilter::Error)
-                    .init()
-                    .unwrap();
-                error!("Unknown log level filter: {}", e);
-            }
-        };
+    if let Some(level) = matches.get_one::<String>("log_level") {
+        SimpleLogger::new()
+            .with_level(LevelFilter::from_str(level).unwrap())
+            .init()
+            .unwrap();
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+    let config = parse_toml(matches.get_one::<String>("config").unwrap());
 
     run(config);
 }

src/mastodon.rs

@@ -2,89 +2,189 @@ use crate::config::MastodonConfig;
 
 use egg_mode::{
     entities::{MentionEntity, UrlEntity},
-    tweet::Tweet,
+    user::UserEntityDetail,
 };
-use elefren::{apps::App, prelude::*, scopes::Scopes};
-use html_escape::decode_html_entities;
-use std::{borrow::Cow, collections::HashMap, io::stdin};
+use megalodon::{
+    generator,
+    mastodon::Mastodon,
+    megalodon::{AppInputOptions, CredentialsFieldAttribute},
+};
+use regex::Regex;
+use std::{collections::HashMap, io::stdin};
 
 /// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse
-fn twitter_mentions(ums: &[MentionEntity]) -> HashMap<String, String> {
-    ums.iter()
+/// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
+/// as well
+pub fn twitter_mentions(
+    toot: &mut String,
+    ums: &[MentionEntity],
+    masto: &HashMap<String, MastodonConfig>,
+) {
+    let tm: HashMap<String, String> = ums
+        .iter()
         .map(|s| {
             (
                 format!("@{}", s.screen_name),
                 format!("@{}@twitter.com", s.screen_name),
             )
         })
-        .collect()
+        .chain(
+            masto
+                .values()
+                .filter(|s| s.mastodon_screen_name.is_some())
+                .map(|s| {
+                    (
+                        format!("@{}", s.twitter_screen_name),
+                        format!(
+                            "@{}@{}",
+                            s.mastodon_screen_name.as_ref().unwrap(),
+                            s.base.split('/').last().unwrap()
+                        ),
+                    )
+                })
+                .collect::<HashMap<String, String>>(),
+        )
+        .collect();
+
+    for (k, v) in tm {
+        *toot = toot.replace(&k, &v);
+    }
 }
 
-/// Decodes urls from UrlEntities
-fn decode_urls(urls: &[UrlEntity]) -> HashMap<String, String> {
+/// Decodes urls in toot
+pub fn decode_urls(toot: &mut String, urls: &HashMap<String, String>) {
+    for (k, v) in urls {
+        *toot = toot.replace(k, v);
+    }
+}
+
+/// Reassociates source url with destination url for rewritting
+/// this takes a Teet UrlEntity and an optional Regex
+pub fn associate_urls(urls: &[UrlEntity], re: &Option<Regex>) -> HashMap<String, String> {
     urls.iter()
         .filter(|s| s.expanded_url.is_some())
         .map(|s| {
-            (
-                s.url.to_owned(),
-                s.expanded_url.as_deref().unwrap().to_owned(),
-            )
+            (s.url.to_owned(), {
+                let mut def = s.expanded_url.as_deref().unwrap().to_owned();
+
+                if let Some(r) = re {
+                    if r.is_match(s.expanded_url.as_deref().unwrap()) {
+                        def = s.display_url.to_owned();
+                    }
+                }
+
+                def
+            })
         })
-        .collect()
+        .collect::<HashMap<String, String>>()
+}
+
+/// Replaces the commonly used services by mirrors, if asked to
+pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<String, String>) {
+    for val in urls.values_mut() {
+        for (k, v) in alts {
+            *val = val.replace(&format!("/{k}/"), &format!("/{v}/"));
+        }
+    }
+}
+
+/// Finds a Mastodon screen_name/base_url from a MastodonConfig
+pub fn find_mastodon_screen_name_by_twitter_screen_name(
+    twitter_screen_name: &str,
+    masto: &HashMap<String, MastodonConfig>,
+) -> Option<(String, String)> {
+    masto.iter().find_map(|(_, v)| {
+        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
+            Some((
+                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
+                v.base.to_owned(),
+            ))
+        } else {
+            None
+        }
+    })
+}
+
+/// Replaces the original quoted tweet by the corresponding toot
+pub fn replace_tweet_by_toot(
+    urls: &mut HashMap<String, String>,
+    twitter_screen_name: &str,
+    tweet_id: u64,
+    mastodon_screen_name: &str,
+    base_url: &str,
+    toot_id: &str,
+) {
+    for val in urls.values_mut() {
+        if val.to_lowercase().starts_with(&format!(
+            "https://twitter.com/{}/status/{}",
+            twitter_screen_name.to_lowercase(),
+            tweet_id
+        )) {
+            *val = format!("{base_url}/@{mastodon_screen_name}/{toot_id}");
+        }
+    }
 }
 
 /// Gets Mastodon Data
 pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
-        base: Cow::from(masto.base.to_owned()),
-        client_id: Cow::from(masto.client_id.to_owned()),
-        client_secret: Cow::from(masto.client_secret.to_owned()),
-        redirect: Cow::from(masto.redirect.to_owned()),
-        token: Cow::from(masto.token.to_owned()),
-    };
-
-    Mastodon::from(data)
+    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
 }
 
-/// Builds toot text from tweet
-pub fn build_basic_status(tweet: &Tweet) -> String {
-    let mut toot = tweet.text.to_owned();
+/// Gets note from twitter_user description
+pub fn get_note_from_description(t: &Option<String>, urls: &[UrlEntity]) -> Option<String> {
+    t.as_ref().map(|d| {
+        let mut n = d.to_owned();
+        let a_urls = associate_urls(urls, &None);
+        decode_urls(&mut n, &a_urls);
+        n
+    })
+}
 
-    for decoded_url in decode_urls(&tweet.entities.urls) {
-        toot = toot.replace(&decoded_url.0, &decoded_url.1);
-    }
-
-    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions) {
-        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
-    }
-
-    decode_html_entities(&toot).to_string()
+/// Gets fields_attribute from UserEntityDetail
+pub fn get_attribute_from_url(
+    user_entity_detail: &Option<UserEntityDetail>,
+) -> Option<Vec<CredentialsFieldAttribute>> {
+    user_entity_detail.as_ref().and_then(|u| {
+        u.urls.first().and_then(|v| {
+            v.expanded_url.as_ref().map(|e| {
+                vec![CredentialsFieldAttribute {
+                    name: v.display_url.to_string(),
+                    value: e.to_string(),
+                }]
+            })
+        })
+    })
 }
 
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str) {
-    let mut builder = App::builder();
-    builder
-        .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
-        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
-        .scopes(Scopes::write_all())
-        .website(Cow::from(
-            "https://framagit.org/veretcle/scootaloo".to_string(),
-        ));
+#[tokio::main]
+pub async fn register(host: &str, screen_name: &str) {
+    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
 
-    let app = builder.build().expect("Cannot build the app");
+    let options = AppInputOptions {
+        redirect_uris: None,
+        scopes: Some(
+            [
+                "read:accounts".to_string(),
+                "write:accounts".to_string(),
+                "write:media".to_string(),
+                "write:statuses".to_string(),
+            ]
+            .to_vec(),
+        ),
+        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
+    };
 
-    let registration = Registration::new(host)
-        .register(app)
-        .expect("Cannot build registration object");
-    let url = registration
-        .authorize_url()
-        .expect("Cannot generate registration URI!");
+    let app_data = mastodon
+        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
+        .await
+        .expect("Cannot build registration object!");
 
-    println!("Click this link to authorize on Mastodon: {}", url);
+    let url = app_data.url.expect("Cannot generate registration URI!");
+
+    println!("Click this link to authorize on Mastodon: {url}");
     println!("Paste the returned authorization code: ");
 
     let mut input = String::new();
@@ -92,16 +192,47 @@ pub fn register(host: &str) {
         .read_line(&mut input)
         .expect("Unable to read back registration code!");
 
-    let code = input.trim();
-    let mastodon = registration
-        .complete(code)
+    let token_data = mastodon
+        .fetch_access_token(
+            app_data.client_id.to_owned(),
+            app_data.client_secret.to_owned(),
+            input.trim().to_string(),
+            megalodon::default::NO_REDIRECT.to_string(),
+        )
+        .await
         .expect("Unable to create access token!");
 
-    let toml = toml::to_string(&*mastodon).unwrap();
+    let mastodon = generator(
+        megalodon::SNS::Mastodon,
+        host.to_string(),
+        Some(token_data.access_token.to_owned()),
+        None,
+    );
+
+    let current_account = mastodon
+        .verify_account_credentials()
+        .await
+        .expect("Unable to access account information!")
+        .json();
 
     println!(
-        "Please insert the following block at the end of your configuration file:\n[mastodon]\n{}",
-        toml
+        r#"Please insert the following block at the end of your configuration file:
+[mastodon.{}]
+twitter_screen_name = "{}"
+mastodon_screen_name = "{}"
+base = "{}"
+client_id = "{}"
+client_secret = "{}"
+redirect = "{}"
+token = "{}""#,
+        screen_name.to_lowercase(),
+        screen_name,
+        current_account.username,
+        host,
+        app_data.client_id,
+        app_data.client_secret,
+        app_data.redirect_uri,
+        token_data.access_token,
     );
 }
 
@@ -109,115 +240,343 @@ pub fn register(host: &str) {
 mod tests {
     use super::*;
 
-    use chrono::prelude::*;
-    use egg_mode::tweet::TweetEntities;
-
     #[test]
-    fn test_twitter_mentions() {
-        let mention_entity = MentionEntity {
-            id: 12345,
-            range: (1, 3),
-            name: "Ta Mere l0l".to_string(),
-            screen_name: "tamerelol".to_string(),
+    fn test_get_attribute_from_url() {
+        let expected_credentials_field_attribute = CredentialsFieldAttribute {
+            name: "Nintendojo.fr".to_string(),
+            value: "https://www.nintendojo.fr".to_string(),
         };
 
-        let twitter_ums = vec![mention_entity];
+        let true_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr".to_string()),
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
 
-        let mut expected_mentions = HashMap::new();
-        expected_mentions.insert(
-            "@tamerelol".to_string(),
-            "@tamerelol@twitter.com".to_string(),
+        let false_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: None,
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        assert!(get_attribute_from_url(&None).is_none());
+        assert!(get_attribute_from_url(&Some(UserEntityDetail { urls: false_urls })).is_none());
+
+        let binding = get_attribute_from_url(&Some(UserEntityDetail { urls: true_urls })).unwrap();
+        let result_credentials_field_attribute = binding.first().unwrap();
+
+        assert_eq!(
+            result_credentials_field_attribute.name,
+            expected_credentials_field_attribute.name
+        );
+        assert_eq!(
+            result_credentials_field_attribute.value,
+            expected_credentials_field_attribute.value
         );
-
-        let decoded_mentions = twitter_mentions(&twitter_ums);
-
-        assert_eq!(expected_mentions, decoded_mentions);
     }
 
     #[test]
-    fn test_decode_urls() {
-        let url_entity1 = UrlEntity {
+    fn test_get_note_from_description() {
+        let urls = vec![UrlEntity {
             display_url: "tamerelol".to_string(),
             expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
             range: (1, 3),
             url: "https://t.me/tamerelol".to_string(),
-        };
+        }];
 
-        let url_entity2 = UrlEntity {
-            display_url: "tamerelol".to_string(),
-            expanded_url: None,
-            range: (1, 3),
-            url: "https://t.me/tamerelol".to_string(),
-        };
+        let some_description = Some("Youpi | https://t.me/tamerelol".to_string());
+        let none_description = None;
 
-        let twitter_urls = vec![url_entity1, url_entity2];
-
-        let mut expected_urls = HashMap::new();
-        expected_urls.insert(
-            "https://t.me/tamerelol".to_string(),
-            "https://www.nintendojo.fr/dojobar".to_string(),
+        assert_eq!(
+            get_note_from_description(&some_description, &urls),
+            Some("Youpi | https://www.nintendojo.fr/dojobar".to_string())
         );
 
-        let decoded_urls = decode_urls(&twitter_urls);
-
-        assert_eq!(expected_urls, decoded_urls);
+        assert_eq!(get_note_from_description(&none_description, &urls), None);
     }
 
     #[test]
-    fn test_build_basic_status() {
-        let t = Tweet {
-            coordinates: None,
-            created_at: Utc::now(),
-            current_user_retweet: None,
-            display_text_range: None,
-            entities: TweetEntities {
-                hashtags: vec![],
-                symbols: vec![],
-                urls: vec![
-                        UrlEntity {
-                            display_url: "youtube.com/watch?v=w5TrSa…".to_string(),
-                            expanded_url: Some("https://www.youtube.com/watch?v=w5TrSaoYmZ8".to_string()),
-                            range: (93, 116),
-                            url: "https://t.co/zXw0FfX2Nt".to_string(),
-                        }
-                ],
-                user_mentions: vec![
-                        MentionEntity {
-                            id: 491500016,
-                            range: (80, 95),
-                            name: "Nintendo France".to_string(),
-                            screen_name: "NintendoFrance".to_string(),
-                        }
-                    ],
-                media: None,
+    fn test_replace_tweet_by_toot() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
+            ),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+        ]);
+
+        replace_tweet_by_toot(
+            &mut associated_urls,
+            "NintendojoFR",
+            1590047921633755136,
+            "nintendojofr",
+            "https://m.nintendojo.fr",
+            "109309605486908797",
+        );
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
+        let masto_config = HashMap::from([
+            (
+                "test".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tonpere".to_string(),
+                    mastodon_screen_name: Some("lalali".to_string()),
+                    twitter_page_size: None,
+                    base: "https://mstdn.net".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test2".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tamerelol".to_string(),
+                    mastodon_screen_name: None,
+                    twitter_page_size: None,
+                    base: "https://mastoot.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test3".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "NintendojoFR".to_string(),
+                    mastodon_screen_name: Some("nintendojofr".to_string()),
+                    twitter_page_size: None,
+                    base: "https://m.nintendojo.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+        ]);
+
+        // case sensitiveness, to avoid any mistake
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
+        );
+        assert_eq!(
+            Some((
+                "nintendojofr".to_string(),
+                "https://m.nintendojo.fr".to_string()
+            )),
+            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
+        );
+        // should return None if twitter_screen_name is undefined
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
+        );
+        assert_eq!(
+            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
+            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
+        );
+    }
+
+    #[test]
+    fn test_twitter_mentions() {
+        let mention_entities = vec![
+            MentionEntity {
+                id: 12345,
+                range: (1, 3),
+                name: "Ta Mere l0l".to_string(),
+                screen_name: "tamerelol".to_string(),
             },
-            extended_entities: None,
-            favorite_count: 0,
-            favorited: None,
-            filter_level: None,
-            id: 1491541246984306693,
-            in_reply_to_user_id: None,
-            in_reply_to_screen_name: None,
-            in_reply_to_status_id: None,
-            lang: None,
-            place: None,
-            possibly_sensitive: None,
-            quoted_status: None,
-            quoted_status_id: None,
-            retweet_count: 0,
-            retweeted: None,
-            retweeted_status: None,
-            source: None,
-            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance https://t.co/zXw0FfX2Nt".to_string(),
-            truncated: false,
-            user: None,
-            withheld_copyright: false,
-            withheld_in_countries: None,
-            withheld_scope: None,
-        };
+            MentionEntity {
+                id: 6789,
+                range: (1, 3),
+                name: "TONPERE".to_string(),
+                screen_name: "tonpere".to_string(),
+            },
+        ];
 
-        let t_out = build_basic_status(&t);
+        let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
 
-        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com https://www.youtube.com/watch?v=w5TrSaoYmZ8");
+        let masto_config = HashMap::from([(
+            "test".to_string(),
+            (MastodonConfig {
+                twitter_screen_name: "tonpere".to_string(),
+                mastodon_screen_name: Some("lalali".to_string()),
+                twitter_page_size: None,
+                base: "https://mstdn.net".to_string(),
+                client_id: "".to_string(),
+                client_secret: "".to_string(),
+                redirect: "".to_string(),
+                token: "".to_string(),
+            }),
+        )]);
+
+        twitter_mentions(&mut toot, &mention_entities, &masto_config);
+
+        assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
+    }
+
+    #[test]
+    fn test_decode_urls() {
+        let urls = HashMap::from([
+            (
+                "https://t.co/thisisatest".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/nopenotinclusive".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
+
+        let mut toot =
+            "Rendez-vous sur https://t.co/thisisatest avec nos https://t.co/nopenotinclusive !"
+                .to_string();
+
+        decode_urls(&mut toot, &urls);
+
+        assert_eq!(
+            &toot,
+            "Rendez-vous sur https://www.nintendojo.fr/dojobar avec nos invité.es !"
+        );
+    }
+
+    #[test]
+    fn test_associate_urls() {
+        let urls = vec![
+            UrlEntity {
+                display_url: "tamerelol".to_string(),
+                expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+                range: (1, 3),
+                url: "https://t.me/tamerelol".to_string(),
+            },
+            UrlEntity {
+                display_url: "sadcat".to_string(),
+                expanded_url: None,
+                range: (1, 3),
+                url: "https://t.me/sadcat".to_string(),
+            },
+            UrlEntity {
+                display_url: "invité.es".to_string(),
+                expanded_url: Some("http://xn--invit-fsa.es".to_string()),
+                range: (85, 108),
+                url: "https://t.co/WAUgnpHLmo".to_string(),
+            },
+        ];
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.me/tamerelol".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/WAUgnpHLmo".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
+
+        let re = Regex::new("(.+)\\.es$").ok();
+
+        let associated_urls = associate_urls(&urls, &re);
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_replace_alt_services() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://www.youtube.com/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://twitter.com/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        let alt_services = HashMap::from([
+            ("twitter.com".to_string(), "nitter.net".to_string()),
+            ("youtu.be".to_string(), "invidio.us".to_string()),
+            ("www.youtube.com".to_string(), "invidio.us".to_string()),
+            ("youtube.com".to_string(), "invidio.us".to_string()),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://invidio.us/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://nitter.net/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        replace_alt_services(&mut associated_urls, &alt_services);
+
+        assert_eq!(associated_urls, expected_urls);
     }
 }

src/state.rs

@@ -1,10 +1,13 @@
-use log::debug;
-use rusqlite::{params, Connection, OptionalExtension};
 use std::error::Error;
 
+use log::debug;
+
+use rusqlite::{params, Connection, OptionalExtension};
+
 /// Struct for each query line
 #[derive(Debug)]
 pub struct TweetToToot {
+    pub twitter_screen_name: String,
     pub tweet_id: u64,
     pub toot_id: String,
 }
@@ -13,12 +16,13 @@ pub struct TweetToToot {
 /// if a tweet_id is passed, read this particular tweet from DB
 pub fn read_state(
     conn: &Connection,
+    n: &str,
     s: Option<u64>,
 ) -> Result<Option<TweetToToot>, Box<dyn Error>> {
     debug!("Reading tweet_id {:?}", s);
     let query: String = match s {
-        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {}", i),
-        None => "SELECT * FROM tweet_to_toot ORDER BY tweet_id DESC LIMIT 1".to_string(),
+        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {i} and twitter_screen_name = \"{n}\""),
+        None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{n}\" ORDER BY tweet_id DESC LIMIT 1"),
     };
 
     let mut stmt = conn.prepare(&query)?;
@@ -26,8 +30,9 @@ pub fn read_state(
     let t = stmt
         .query_row([], |row| {
             Ok(TweetToToot {
-                tweet_id: row.get(0)?,
-                toot_id: row.get(1)?,
+                twitter_screen_name: row.get("twitter_screen_name")?,
+                tweet_id: row.get("tweet_id")?,
+                toot_id: row.get("toot_id")?,
             })
         })
         .optional()?;
@@ -39,8 +44,8 @@ pub fn read_state(
 pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
     debug!("Write struct {:?}", t);
     conn.execute(
-        "INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)",
-        params![t.tweet_id, t.toot_id],
+        "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id) VALUES (?1, ?2, ?3)",
+        params![t.twitter_screen_name, t.tweet_id, t.toot_id],
     )?;
 
     Ok(())
@@ -53,8 +58,9 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
 
     conn.execute(
         "CREATE TABLE IF NOT EXISTS tweet_to_toot (
-            tweet_id        INTEGER PRIMARY KEY,
-            toot_id         TEXT UNIQUE
+            twitter_screen_name TEXT NOT NULL,
+            tweet_id            INTEGER PRIMARY KEY,
+            toot_id             TEXT UNIQUE
             )",
         [],
     )?;
@@ -62,6 +68,30 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+/// Migrate DB from 0.6.x to 0.7.x
+pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
+    debug!("Migrating DB for Scootaloo");
+
+    let conn = Connection::open(d)?;
+
+    let res = conn.execute(
+        &format!(
+            "ALTER TABLE tweet_to_toot
+        ADD COLUMN twitter_screen_name TEXT NOT NULL
+        DEFAULT \"{s}\""
+        ),
+        [],
+    );
+
+    match res {
+        Err(e) => match e.to_string().as_str() {
+            "duplicate column name: twitter_screen_name" => Ok(()),
+            _ => Err(e.into()),
+        },
+        _ => Ok(()),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -93,9 +123,9 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (100, 'A');",
+            ('tamerelol', 100, 'A');",
             [],
         )
         .unwrap();
@@ -114,6 +144,7 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         let t_in = TweetToToot {
+            twitter_screen_name: "tamerelol".to_string(),
             tweet_id: 123456789,
             toot_id: "987654321".to_string(),
         };
@@ -125,14 +156,16 @@ mod tests {
         let t_out = stmt
             .query_row([], |row| {
                 Ok(TweetToToot {
-                    tweet_id: row.get(0).unwrap(),
-                    toot_id: row.get(1).unwrap(),
+                    twitter_screen_name: row.get("twitter_screen_name").unwrap(),
+                    tweet_id: row.get("tweet_id").unwrap(),
+                    toot_id: row.get("toot_id").unwrap(),
                 })
             })
             .unwrap();
 
+        assert_eq!(&t_out.twitter_screen_name, "tamerelol");
         assert_eq!(t_out.tweet_id, 123456789);
-        assert_eq!(t_out.toot_id, "987654321".to_string());
+        assert_eq!(&t_out.toot_id, "987654321");
 
         remove_file(d).unwrap();
     }
@@ -146,15 +179,15 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (101, 'A'),
-            (102, 'B');",
+            ('tamerelol', 101, 'A'),
+            ('tamerelol', 102, 'B');",
             [],
         )
         .unwrap();
 
-        let t_out = read_state(&conn, None).unwrap().unwrap();
+        let t_out = read_state(&conn, "tamerelol", None).unwrap().unwrap();
 
         remove_file(d).unwrap();
 
@@ -170,7 +203,7 @@ mod tests {
 
         let conn = Connection::open(d).unwrap();
 
-        let t_out = read_state(&conn, None).unwrap();
+        let t_out = read_state(&conn, "tamerelol", None).unwrap();
 
         remove_file(d).unwrap();
 
@@ -186,14 +219,14 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (100, 'A');",
+            ('tamerelol', 100, 'A');",
             [],
         )
         .unwrap();
 
-        let t_out = read_state(&conn, Some(101)).unwrap();
+        let t_out = read_state(&conn, "tamerelol", Some(101)).unwrap();
 
         remove_file(d).unwrap();
 
@@ -209,18 +242,62 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (100, 'A');",
+            ('tamerelol', 100, 'A');",
             [],
         )
         .unwrap();
 
-        let t_out = read_state(&conn, Some(100)).unwrap().unwrap();
+        let t_out = read_state(&conn, "tamerelol", Some(100)).unwrap().unwrap();
 
         remove_file(d).unwrap();
 
         assert_eq!(t_out.tweet_id, 100);
         assert_eq!(t_out.toot_id, "A");
     }
+
+    #[test]
+    fn test_migrate_db_add_column() {
+        let d = "/tmp/test_migrate_db_add_column.sqlite";
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS tweet_to_toot (
+            tweet_id            INTEGER PRIMARY KEY,
+            toot_id             TEXT UNIQUE
+            )",
+            [],
+        )
+        .unwrap();
+
+        migrate_db(d, "tamerelol").unwrap();
+
+        let mut stmt = conn.prepare("PRAGMA table_info(tweet_to_toot);").unwrap();
+
+        let mut t = stmt.query([]).unwrap();
+
+        while let Some(row) = t.next().unwrap() {
+            if row.get::<usize, u8>(0).unwrap() == 2 {
+                assert_eq!(
+                    row.get::<usize, String>(1).unwrap(),
+                    "twitter_screen_name".to_string()
+                );
+            }
+        }
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_migrate_db_no_add_column() {
+        let d = "/tmp/test_migrate_db_no_add_column.sqlite";
+
+        init_db(d).unwrap();
+
+        migrate_db(d, "tamerelol").unwrap();
+
+        remove_file(d).unwrap();
+    }
 }

src/twitter.rs

@@ -27,52 +27,167 @@ pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
     }
 }
 
-/// Gets Twitter user timeline
+/// Gets Twitter user timeline, eliminate responses to others and reverse it
 pub async fn get_user_timeline(
-    config: &TwitterConfig,
-    token: Token,
+    screen_name: &str,
+    token: &Token,
     lid: Option<u64>,
+    page_size: i32,
 ) -> Result<Vec<Tweet>, Box<dyn Error>> {
     // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_, feed) = user_timeline(
-        UserID::from(config.username.to_owned()),
-        true,
-        false,
-        &token,
-    )
-    .with_page_size(200)
-    .older(lid)
-    .await?;
+    let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
+        .with_page_size(page_size)
+        .older(lid)
+        .await?;
 
-    Ok(feed.to_vec())
+    let mut feed: Vec<Tweet> = feed
+        .iter()
+        .cloned()
+        .filter(|t| match &t.in_reply_to_screen_name {
+            Some(r) => r.to_lowercase() == screen_name.to_lowercase(),
+            None => true,
+        })
+        .collect();
+
+    feed.reverse();
+
+    Ok(feed)
 }
 
 /// Retrieves a single media from a tweet and store it in a temporary file
 pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
     match m.media_type {
-        MediaType::Photo => {
-            return cache_media(&m.media_url_https, t).await;
-        }
+        MediaType::Photo => cache_media(&m.media_url_https, t).await,
         _ => match &m.video_info {
-            Some(v) => {
-                for variant in &v.variants {
-                    if variant.content_type == "video/mp4" {
-                        return cache_media(&variant.url, t).await;
-                    }
-                }
-                return Err(ScootalooError::new(&format!(
+            Some(v) => match &v.variants.iter().find(|&x| x.content_type == "video/mp4") {
+                Some(u) => cache_media(&u.url, t).await,
+                None => Err(ScootalooError::new(&format!(
                     "Media Type for {} is video but no mp4 file URL is available",
                     &m.url
                 ))
-                .into());
-            }
-            None => {
-                return Err(ScootalooError::new(&format!(
-                    "Media Type for {} is video but does not contain any video_info",
-                    &m.url
-                ))
-                .into());
-            }
+                .into()),
+            },
+            None => Err(ScootalooError::new(&format!(
+                "Media Type for {} is video but does not contain any video_info",
+                &m.url
+            ))
+            .into()),
         },
-    };
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use egg_mode::entities::{
+        MediaSize, MediaSizes,
+        MediaType::{Gif, Photo},
+        ResizeMode::Crop,
+        ResizeMode::Fit,
+        VideoInfo, VideoVariant,
+    };
+    use std::fs::remove_dir_all;
+
+    const TMP_DIR: &'static str = "/tmp/scootaloo_get_tweet_media_test";
+
+    #[tokio::test]
+    async fn test_get_tweet_media() {
+        let m_photo = MediaEntity {
+            display_url: "pic.twitter.com/sHrwmP69Yv".to_string(),
+            expanded_url: "https://twitter.com/NintendojoFR/status/1555473821121056771/photo/1"
+                .to_string(),
+            id: 1555473771280080896,
+            range: (91, 114),
+            media_url: "http://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg".to_string(),
+            media_url_https: "https://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg"
+                .to_string(),
+            sizes: MediaSizes {
+                thumb: MediaSize {
+                    w: 150,
+                    h: 150,
+                    resize: Crop
+                },
+                small: MediaSize {
+                    w: 680,
+                    h: 510,
+                    resize: Fit
+                },
+                medium: MediaSize {
+                    w: 1200,
+                    h: 900,
+                    resize: Fit
+                },
+                large: MediaSize {
+                    w: 1280,
+                    h: 960,
+                    resize: Fit
+                }
+            },
+            source_status_id: None,
+            media_type: Photo,
+            url: "https://t.co/sHrwmP69Yv".to_string(),
+            video_info: None,
+            ext_alt_text: Some("Le menu «\u{a0}Classes » du jeu vidéo Xenoblade Chronicles 3 (Switch). L’affinité du personnage pour la classe est notée par quatre lettres : C, A, C, A (caca)."
+                .to_string())
+        };
+        let m_video = MediaEntity {
+            display_url: "pic.twitter.com/xDln0RrkjU".to_string(),
+            expanded_url: "https://twitter.com/NintendojoFR/status/1551822196833673218/photo/1"
+                .to_string(),
+            id: 1551822189711790081,
+            range: (275, 298),
+            media_url: "http://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg".to_string(),
+            media_url_https: "https://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg"
+                .to_string(),
+            sizes: MediaSizes {
+                thumb: MediaSize {
+                    w: 150,
+                    h: 150,
+                    resize: Crop,
+                },
+                small: MediaSize {
+                    w: 320,
+                    h: 240,
+                    resize: Fit,
+                },
+                medium: MediaSize {
+                    w: 320,
+                    h: 240,
+                    resize: Fit,
+                },
+                large: MediaSize {
+                    w: 320,
+                    h: 240,
+                    resize: Fit,
+                },
+            },
+            source_status_id: None,
+            media_type: Gif,
+            url: "https://t.co/xDln0RrkjU".to_string(),
+            video_info: Some(VideoInfo {
+                aspect_ratio: (4, 3),
+                duration_millis: None,
+                variants: vec![VideoVariant {
+                    bitrate: Some(0),
+                    content_type: "video/mp4".parse::<mime::Mime>().unwrap(),
+                    url: "https://video.twimg.com/tweet_video/FYkuD0RXEAE-iDx.mp4".to_string(),
+                }],
+            }),
+            ext_alt_text: Some("Scared Nintendo GIF".to_string()),
+        };
+
+        let tweet_media_photo = get_tweet_media(&m_photo, TMP_DIR).await.unwrap();
+        let tweet_media_video = get_tweet_media(&m_video, TMP_DIR).await.unwrap();
+
+        assert_eq!(
+            tweet_media_photo,
+            format!("{}/FZYnJ1qWIAAReHt.jpg", TMP_DIR)
+        );
+        assert_eq!(
+            tweet_media_video,
+            format!("{}/FYkuD0RXEAE-iDx.mp4", TMP_DIR)
+        );
+
+        remove_dir_all(TMP_DIR).unwrap();
+    }
 }

src/util.rs

@@ -1,11 +1,127 @@
-use crate::ScootalooError;
+use crate::{twitter::get_tweet_media, ScootalooError};
+
+use base64::encode;
+use egg_mode::tweet::Tweet;
+use futures::{stream, stream::StreamExt};
+use log::{error, info, warn};
+use megalodon::{
+    entities::UploadMedia::{AsyncAttachment, Attachment},
+    error,
+    mastodon::Mastodon,
+    megalodon::Megalodon,
+};
 use reqwest::Url;
 use std::error::Error;
 use tokio::{
-    fs::{create_dir_all, File},
+    fs::{create_dir_all, remove_file, File},
     io::copy,
+    time::{sleep, Duration},
 };
 
+/// Generate associative table between media ids and tweet extended entities
+pub async fn generate_media_ids(
+    tweet: &Tweet,
+    cache_path: &str,
+    mastodon: &Mastodon,
+) -> (String, Vec<String>) {
+    let mut media_url = "".to_string();
+    let mut media_ids: Vec<String> = vec![];
+
+    if let Some(m) = &tweet.extended_entities {
+        info!("{} medias in tweet", m.media.len());
+
+        let medias = m.media.clone();
+
+        let mut stream = stream::iter(medias)
+            .map(|media| {
+                // attribute media url
+                media_url = media.url.clone();
+
+                // clone everything we need
+                let cache_path = String::from(cache_path);
+                let mastodon = mastodon.clone();
+
+                tokio::task::spawn(async move {
+                    info!("Start treating {}", media.media_url_https);
+                    // get the tweet embedded media
+                    let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
+
+                    // upload media to Mastodon
+                    let mastodon_media = mastodon
+                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        .await?
+                        .json();
+                    // at this point, we can safely erase the original file
+                    // it doesn’t matter if we can’t remove, cache_media fn is idempotent
+                    remove_file(&local_tweet_media_path).await.ok();
+
+                    let id = match mastodon_media {
+                        Attachment(m) => m.id,
+                        AsyncAttachment(m) => wait_until_uploaded(&mastodon, &m.id).await?,
+                    };
+
+                    Ok::<String, ScootalooError>(id)
+                })
+            })
+            .buffered(4); // there are max four medias per tweet and they need to be treated in
+                          // order
+
+        while let Some(result) = stream.next().await {
+            match result {
+                Ok(Ok(v)) => media_ids.push(v),
+                Ok(Err(e)) => warn!("Cannot treat media: {}", e),
+                Err(e) => error!("Something went wrong when joining the main thread: {}", e),
+            }
+        }
+    } else {
+        info!("No media in tweet");
+    }
+
+    // in case some media_ids slot remained empty due to errors, remove them
+    media_ids.retain(|x| !x.is_empty());
+
+    (media_url, media_ids)
+}
+
+/// Wait on uploaded medias when necessary
+async fn wait_until_uploaded(client: &Mastodon, id: &str) -> Result<String, error::Error> {
+    loop {
+        sleep(Duration::from_secs(1)).await;
+        let res = client.get_media(id.to_string()).await;
+        return match res {
+            Ok(res) => Ok(res.json.id),
+            Err(err) => match err {
+                error::Error::OwnError(ref own_err) => match own_err.kind {
+                    error::Kind::HTTPPartialContentError => continue,
+                    _ => Err(err),
+                },
+                _ => Err(err),
+            },
+        };
+    }
+}
+
+/// Transforms the media into a base64 equivalent
+pub async fn base64_media(u: &str) -> Result<String, Box<dyn Error>> {
+    let mut response = reqwest::get(u).await?;
+
+    let mut buffer = Vec::new();
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut buffer).await?;
+    }
+
+    let content_type = response
+        .headers()
+        .get("content-type")
+        .ok_or_else(|| ScootalooError::new(&format!("Cannot get media content type for {u}")))?
+        .to_str()?;
+
+    let encoded_f = encode(buffer);
+
+    Ok(format!("data:{content_type};base64,{encoded_f}"))
+}
+
 /// Gets and caches Twitter Media inside the determined temp dir
 pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
     // create dir
@@ -20,19 +136,17 @@ pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
         .path_segments()
         .ok_or_else(|| {
             ScootalooError::new(&format!(
-                "Cannot determine the destination filename for {}",
-                u
+                "Cannot determine the destination filename for {u}"
             ))
         })?
         .last()
         .ok_or_else(|| {
             ScootalooError::new(&format!(
-                "Cannot determine the destination filename for {}",
-                u
+                "Cannot determine the destination filename for {u}"
             ))
         })?;
 
-    let dest_filepath = format!("{}/{}", t, dest_filename);
+    let dest_filepath = format!("{t}/{dest_filename}");
 
     let mut dest_file = File::create(&dest_filepath).await?;
 
@@ -64,4 +178,17 @@ mod tests {
 
         remove_dir_all(TMP_DIR).unwrap();
     }
+
+    #[tokio::test]
+    async fn test_base64_media() {
+        let img = base64_media(
+            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
+        )
+        .await
+        .unwrap();
+
+        assert!(img.starts_with("data:image/png;base64,"));
+
+        assert!(img.ends_with("="));
+    }
 }

tests/config.rs

@@ -1,4 +1,108 @@
 use scootaloo::parse_toml;
+use std::collections::HashMap;
+
+#[test]
+fn test_alt_services() {
+    let toml = parse_toml("tests/no_test_alt_services.toml");
+    assert_eq!(toml.scootaloo.alternative_services_for, None);
+
+    let toml = parse_toml("tests/test_alt_services.toml");
+    assert_eq!(
+        toml.scootaloo.alternative_services_for,
+        Some(HashMap::from([
+            ("tamere.lol".to_string(), "tonpere.mdr".to_string()),
+            ("you.pi".to_string(), "you.pla".to_string())
+        ]))
+    );
+}
+
+#[test]
+fn test_re_display() {
+    let toml = parse_toml("tests/no_show_url_as_display_url_for.toml");
+    assert_eq!(toml.scootaloo.show_url_as_display_url_for, None);
+
+    let toml = parse_toml("tests/show_url_as_display_url_for.toml");
+
+    assert_eq!(
+        toml.scootaloo.show_url_as_display_url_for,
+        Some("^(.+)\\.es$".to_string())
+    );
+}
+
+#[test]
+fn test_page_size() {
+    const DEFAULT_PAGE_SIZE: i32 = 200;
+    let toml = parse_toml("tests/page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, Some(100));
+
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42));
+
+    // this is the exact line that is used inside fn run() to determine the twitter page size
+    // passed to fn get_user_timeline()
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+    let page_size_for_1 = toml
+        .mastodon
+        .get("1")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 100);
+    assert_eq!(page_size_for_1, 42);
+
+    let toml = parse_toml("tests/no_page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, None);
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    // and same here
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 200);
+}
+
+#[test]
+fn test_parse_good_toml_rate_limit() {
+    let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml");
+
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize));
+}
+
+#[test]
+fn test_parse_good_toml_mastodon_screen_name() {
+    let parse_good_toml = parse_toml("tests/good_test_mastodon_screen_name.toml");
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("0")
+            .unwrap()
+            .mastodon_screen_name,
+        Some("tarace".to_string())
+    );
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("1")
+            .unwrap()
+            .mastodon_screen_name,
+        None
+    );
+}
 
 #[test]
 fn test_parse_good_toml() {
@@ -9,21 +113,45 @@ fn test_parse_good_toml() {
         "/var/random/scootaloo.sqlite"
     );
     assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, None);
 
-    assert_eq!(parse_good_toml.twitter.username, "tamerelol");
     assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
     assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");
     assert_eq!(parse_good_toml.twitter.access_key, "rand access key");
     assert_eq!(parse_good_toml.twitter.access_secret, "super secret");
 
-    assert_eq!(parse_good_toml.mastodon.base, "https://m.nintendojo.fr");
-    assert_eq!(parse_good_toml.mastodon.client_id, "rand client id");
-    assert_eq!(parse_good_toml.mastodon.client_secret, "secret");
     assert_eq!(
-        parse_good_toml.mastodon.redirect,
+        &parse_good_toml
+            .mastodon
+            .get("tamerelol")
+            .unwrap()
+            .twitter_screen_name,
+        "tamerelol"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().base,
+        "https://m.nintendojo.fr"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().client_id,
+        "rand client id"
+    );
+    assert_eq!(
+        &parse_good_toml
+            .mastodon
+            .get("tamerelol")
+            .unwrap()
+            .client_secret,
+        "secret"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().redirect,
         "urn:ietf:wg:oauth:2.0:oob"
     );
-    assert_eq!(parse_good_toml.mastodon.token, "super secret");
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().token,
+        "super secret"
+    );
 }
 
 #[test]

tests/good_test.toml

@@ -4,14 +4,14 @@ db_path="/var/random/scootaloo.sqlite"
 cache_path="/tmp/scootaloo"
 
 [twitter]
-username="tamerelol"
-
 consumer_key="rand consumer key"
 consumer_secret="secret"
 access_key="rand access key"
 access_secret="super secret"
 
 [mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
 base = "https://m.nintendojo.fr"
 client_id = "rand client id"
 client_secret = "secret"

tests/good_test_mastodon_screen_name.toml

@@ -0,0 +1,28 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+mastodon_screen_name="tarace"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/good_test_rate_limit.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+rate_limit=69
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_page_size.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_show_url_as_display_url_for.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_test_alt_services.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/page_size.toml

@@ -0,0 +1,29 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+page_size=100
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tonperemdr"
+twitter_page_size=42
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/show_url_as_display_url_for.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+show_url_as_display_url_for = "^(.+)\\.es$"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/test_alt_services.toml

@@ -0,0 +1,22 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr"
+"you.pi" = "you.pla"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"