Update README.md

Update file README.md

See merge request veretcle/scootaloo!57

Cargo.toml

@@ -1,21 +1,22 @@
 [package]
 name = "scootaloo"
-version = "0.8.0"
+version = "1.1.6"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-chrono = "^0.4"
+base64 = "^0.13"
+regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^2.34"
-futures = "^0.3"
+clap = "^4"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-tokio = { version = "1", features = ["full"]}
-elefren = "^0.22"
+tokio = { version = "^1", features = ["rt"]}
+futures = "^0.3"
+megalodon = "^0.3.6"
 html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
@@ -24,3 +25,5 @@ mime = "^0.3"
 
 [profile.release]
 strip = true
+lto = true
+codegen-units = 1

README.md

@@ -1,3 +1,9 @@
+**Due to the new Twitter policy about API v1.1 and API v2, this project will no longer be updated as it no longer can work for free any way, shape or form.**
+
+First of all, I’m deeply sorry about that. It worked pretty great for what it did with a level of quality I’m very proud to have achieved.
+
+Secondly, fuck you Musk, fuck you.
+
 A Twitter to Mastodon copy bot written in Rust
 
 It:
@@ -18,9 +24,24 @@ First up, create a configuration file (default path is `/usr/local/etc/scootaloo
 [scootaloo]
 db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
 cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneously
+## optional, this should be omitted the majority of the time
+## sometimes, twitter try to use french inclusive writting, but instead of using `·` (median point), they’re using `.`
+## this makes twitter interpret it as a URL, which is wrong
+## this parameter allows you to catch such URLs and apply the `display_url` (i.e. `tout.es`) instead of the `expanded_url` (i.e. `http://tout.es`)
+## in those particular cases
+## (!) use with caution, it might have some undesired effects
+show_url_as_display_url_for = "^https?://(.+)\\.es$"
+## optional, this allows you to replace the host for popular services such as YouTube of Twitter, or any other
+## with their more freely accessible equivalent
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr" ## quotes are necessary for both parameters
+"you.pi" = "you.pla"
+"www.you.pi" = "you.pla" ## this is an exact match, so you’ll need to lay out all the possibilities
 
 [twitter]
 ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
+page_size = 20 ## optional, default 200, max number of tweet retrieved
 consumer_key = "MYCONSUMERKEY"
 consumer_secret = "MYCONSUMERSECRET"
 access_key = "MYACCESSKEY"
@@ -45,7 +66,8 @@ This will give you the end of the TOML file. It will look like this:
 
 ```toml
 [mastodon.nintendojofr] ## account
-twitter_screen_name="NintendojoFR" ## User Timeline to copy
+twitter_screen_name = "NintendojoFR" ## User Timeline to copy
+mastodon_screen_name = "nintendojofr" ## optional, Mastodon account name used for smart mentions
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
 client_secret = "MYCLIENTSECRET"
@@ -55,6 +77,11 @@ token = "MYTOKEN"
 
 You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
 
+You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so:
+```
+twitter_page_size = 40
+```
+
 ## Running
 
 You can then run the application via `cron` for example. Here is the generic usage:
@@ -62,22 +89,20 @@ You can then run the application via `cron` for example. Here is the generic usa
 ```sh
 A Twitter to Mastodon bot
 
-USAGE:
-    scootaloo [OPTIONS] [SUBCOMMAND]
+Usage: scootaloo [OPTIONS] [COMMAND]
 
-FLAGS:
-    -h, --help       Prints help information
-    -V, --version    Prints version information
-
-OPTIONS:
-    -c, --config <CONFIG_FILE>    TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)
-    -l, --loglevel <LOGLEVEL>     Log level. Valid values are: Off, Warn, Error, Info, Debug
-
-SUBCOMMANDS:
-    help        Prints this message or the help of the given subcommand(s)
+Commands:
+  register     Command to register to a Mastodon Instance
   init         Command to init Scootaloo DB
   migrate      Command to migrate Scootaloo DB
-    register    Command to register to a Mastodon Instance
+  copyprofile  Command to copy a Twitter profile into Mastodon
+  help         Print this message or the help of the given subcommand(s)
+
+Options:
+  -c, --config <CONFIG_FILE>  TOML config file for scootaloo [default: /usr/local/etc/scootaloo.toml]
+  -l, --loglevel <LOGLEVEL>   Log level [possible values: Off, Warn, Error, Info, Debug]
+  -h, --help                  Print help information
+  -V, --version               Print version information
 ```
 
 # Quirks

src/config.rs

@@ -16,11 +16,14 @@ pub struct TwitterConfig {
     pub consumer_secret: String,
     pub access_key: String,
     pub access_secret: String,
+    pub page_size: Option<i32>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
 pub struct MastodonConfig {
     pub twitter_screen_name: String,
+    pub mastodon_screen_name: Option<String>,
+    pub twitter_page_size: Option<i32>,
     pub base: String,
     pub client_id: String,
     pub client_secret: String,
@@ -32,15 +35,18 @@ pub struct MastodonConfig {
 pub struct ScootalooConfig {
     pub db_path: String,
     pub cache_path: String,
+    pub rate_limit: Option<usize>,
+    pub show_url_as_display_url_for: Option<String>,
+    pub alternative_services_for: Option<HashMap<String, String>>,
 }
 
 /// Parses the TOML file into a Config Struct
 pub fn parse_toml(toml_file: &str) -> Config {
     let toml_config = read_to_string(toml_file)
-        .unwrap_or_else(|e| panic!("Cannot open config file {}: {}", toml_file, e));
+        .unwrap_or_else(|e| panic!("Cannot open config file {toml_file}: {e}"));
 
     let config: Config = toml::from_str(&toml_config)
-        .unwrap_or_else(|e| panic!("Cannot parse TOML file {}: {}", toml_file, e));
+        .unwrap_or_else(|e| panic!("Cannot parse TOML file {toml_file}: {e}"));
 
     config
 }

src/error.rs

@@ -1,8 +1,12 @@
 use std::{
+    boxed::Box,
+    convert::From,
     error::Error,
     fmt::{Display, Formatter, Result},
 };
 
+use megalodon::error::Error as megalodonError;
+
 #[derive(Debug)]
 pub struct ScootalooError {
     details: String,
@@ -23,3 +27,15 @@ impl Display for ScootalooError {
         write!(f, "{}", self.details)
     }
 }
+
+impl From<Box<dyn Error>> for ScootalooError {
+    fn from(error: Box<dyn Error>) -> Self {
+        ScootalooError::new(&format!("Error in a subset crate: {error}"))
+    }
+}
+
+impl From<megalodonError> for ScootalooError {
+    fn from(error: megalodonError) -> Self {
+        ScootalooError::new(&format!("Error in megalodon crate: {error}"))
+    }
+}

src/lib.rs

@@ -7,30 +7,35 @@ use config::Config;
 
 mod mastodon;
 pub use mastodon::register;
-use mastodon::{build_basic_status, get_mastodon_token};
+use mastodon::*;
 
 mod twitter;
 use twitter::*;
 
 mod util;
-use crate::util::generate_media_ids;
+use util::{base64_media, generate_media_ids};
 
 mod state;
 pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
-use elefren::{prelude::*, status_builder::StatusBuilder};
+use futures::StreamExt;
+use html_escape::decode_html_entities;
 use log::info;
+use megalodon::{
+    megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
+};
+use regex::Regex;
 use rusqlite::Connection;
 use std::sync::Arc;
 use tokio::{spawn, sync::Mutex};
 
+const DEFAULT_RATE_LIMIT: usize = 4;
+const DEFAULT_PAGE_SIZE: i32 = 200;
+
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
-    // create the task vector for handling multiple accounts
-    let mut mtask = vec![];
-
     // open the SQLite connection
     let conn = Arc::new(Mutex::new(
         Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
@@ -41,71 +46,119 @@ pub async fn run(config: Config) {
         }),
     ));
 
-    for mastodon_config in config.mastodon.into_values() {
+    let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
+
+    let display_url_re = config
+        .scootaloo
+        .show_url_as_display_url_for
+        .as_ref()
+        .map(|r|
+        // we want to panic in case the RE is not valid
+        Regex::new(r).unwrap());
+
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            // calculate Twitter page size
+            let page_size = mastodon_config
+                .twitter_page_size
+                .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
             // create temporary value for each task
             let scootaloo_cache_path = config.scootaloo.cache_path.clone();
+            let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
+            let display_url_re = display_url_re.clone();
             let token = get_oauth2_token(&config.twitter);
             let task_conn = conn.clone();
+            let global_mastodon_config = global_mastodon_config.clone();
 
-        let task = spawn(async move {
+            spawn(async move {
                 info!("Starting treating {}", &mastodon_config.twitter_screen_name);
-
                 // retrieve the last tweet ID for the username
                 let lconn = task_conn.lock().await;
-            let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)
-                .unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
-                .map(|s| s.tweet_id);
+                let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
+                    .map(|r| r.tweet_id);
                 drop(lconn);
 
+                // get reversed, curated user timeline
+                let feed = get_user_timeline(
+                    &mastodon_config.twitter_screen_name,
+                    &token,
+                    last_tweet_id,
+                    page_size,
+                )
+                .await?;
+
                 // get Mastodon instance
                 let mastodon = get_mastodon_token(&mastodon_config);
 
-            // get user timeline feed (Vec<tweet>)
-            let mut feed =
-                get_user_timeline(&mastodon_config.twitter_screen_name, &token, last_tweet_id)
-                    .await
-                    .unwrap_or_else(|e| {
-                        panic!(
-                            "Something went wrong when trying to retrieve {}’s timeline: {}",
-                            &mastodon_config.twitter_screen_name, e
-                        )
-                    });
-
-            // empty feed -> exiting
-            if feed.is_empty() {
-                info!("Nothing to retrieve since last time, exiting…");
-                return;
-            }
-
-            // order needs to be chronological
-            feed.reverse();
-
                 for tweet in &feed {
                     info!("Treating Tweet {} inside feed", tweet.id);
-                // initiate the toot_reply_id var
-                let mut toot_reply_id: Option<String> = None;
-                // determine if the tweet is part of a thread (response to self) or a standard response
-                if let Some(r) = &tweet.in_reply_to_screen_name {
-                    if r.to_lowercase() != mastodon_config.twitter_screen_name.to_lowercase() {
-                        // we are responding not threading
-                        info!("Tweet is a direct response, skipping");
-                        continue;
-                    }
-                    info!("Tweet is a thread");
-                    // get the corresponding toot id
-                    let lconn = task_conn.lock().await;
-                    toot_reply_id = read_state(
-                        &lconn,
-                        &mastodon_config.twitter_screen_name,
-                        tweet.in_reply_to_status_id,
-                    )
-                    .unwrap_or(None)
-                    .map(|s| s.toot_id);
-                    drop(lconn);
-                };
 
-                // build basic status by just yielding text and dereferencing contained urls
-                let mut status_text = build_basic_status(tweet);
+                    // basic toot text
+                    let mut status_text = tweet.text.clone();
+
+                    // add mentions and smart mentions
+                    if !&tweet.entities.user_mentions.is_empty() {
+                        info!("Tweet contains mentions, add them!");
+                        let global_mastodon_config = global_mastodon_config.lock().await;
+                        twitter_mentions(
+                            &mut status_text,
+                            &tweet.entities.user_mentions,
+                            &global_mastodon_config,
+                        );
+                        drop(global_mastodon_config);
+                    }
+
+                    if !&tweet.entities.urls.is_empty() {
+                        info!("Tweet contains links, add them!");
+                        let mut associated_urls =
+                            associate_urls(&tweet.entities.urls, &display_url_re);
+
+                        if let Some(q) = &tweet.quoted_status {
+                            if let Some(u) = &q.user {
+                                info!(
+                                    "Tweet {} contains a quote, we try to find it within the DB",
+                                    tweet.id
+                                );
+                                // we know we have a quote and a user, we can lock both the
+                                // connection to DB and global_config
+                                // we will release them manually as soon as they’re useless
+                                let lconn = task_conn.lock().await;
+                                let global_mastodon_config = global_mastodon_config.lock().await;
+                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
+                                {
+                                    info!("We have found the associated toot({})", &r.toot_id);
+                                    // drop conn immediately after the request: we won’t need it
+                                    // any more and the treatment there might be time-consuming
+                                    drop(lconn);
+                                    if let Some((m, t)) =
+                                        find_mastodon_screen_name_by_twitter_screen_name(
+                                            &r.twitter_screen_name,
+                                            &global_mastodon_config,
+                                        )
+                                    {
+                                        // drop the global conf, we have all we required, no need
+                                        // to block it further
+                                        drop(global_mastodon_config);
+                                        replace_tweet_by_toot(
+                                            &mut associated_urls,
+                                            &r.twitter_screen_name,
+                                            q.id,
+                                            &m,
+                                            &t,
+                                            &r.toot_id,
+                                        );
+                                    }
+                                }
+                            }
+                        }
+
+                        if let Some(a) = &scootaloo_alt_services {
+                            replace_alt_services(&mut associated_urls, a);
+                        }
+
+                        decode_urls(&mut status_text, &associated_urls);
+                    }
 
                     // building associative media list
                     let (media_url, status_medias) =
@@ -113,28 +166,55 @@ pub async fn run(config: Config) {
 
                     status_text = status_text.replace(&media_url, "");
 
+                    // now that the text won’t be altered anymore, we can safely remove HTML
+                    // entities
+                    status_text = decode_html_entities(&status_text).to_string();
+
                     info!("Building corresponding Mastodon status");
 
-                let mut status_builder = StatusBuilder::new();
+                    let mut post_status = PostStatusInputOptions {
+                        media_ids: None,
+                        poll: None,
+                        in_reply_to_id: None,
+                        sensitive: None,
+                        spoiler_text: None,
+                        visibility: None,
+                        scheduled_at: None,
+                        language: None,
+                        quote_id: None,
+                    };
 
-                status_builder.status(&status_text).media_ids(status_medias);
+                    if !status_medias.is_empty() {
+                        post_status.media_ids = Some(status_medias);
+                    }
 
-                if let Some(i) = toot_reply_id {
-                    status_builder.in_reply_to(&i);
+                    // thread if necessary
+                    if tweet.in_reply_to_user_id.is_some() {
+                        let lconn = task_conn.lock().await;
+                        if let Ok(Some(r)) = read_state(
+                            &lconn,
+                            &mastodon_config.twitter_screen_name,
+                            tweet.in_reply_to_status_id,
+                        ) {
+                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
+                        }
+                        drop(lconn);
+                    }
+
+                    // language if any
+                    if let Some(l) = &tweet.lang {
+                        post_status.language = Some(l.to_string());
                     }
 
                     // can be activated for test purposes
-                // status_builder.visibility(elefren::status_builder::Visibility::Private);
+                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
 
-                let status = status_builder
-                    .build()
-                    .unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
-
-                // publish status
-                // again unwrap is safe here as we are in the main thread
-                let published_status = mastodon.new_status(status).unwrap();
-                // this will panic if it cannot publish the status, which is a good thing, it allows the
-                // last_tweet gathered not to be written
+                    let published_status = mastodon
+                        .post_status(status_text, Some(&post_status))
+                        .await?
+                        .json();
+                    // this will return if it cannot publish the status preventing the last_tweet from
+                    // being written into db
 
                     let ttt_towrite = TweetToToot {
                         twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
@@ -144,18 +224,89 @@ pub async fn run(config: Config) {
 
                     // write the current state (tweet ID and toot ID) to avoid copying it another time
                     let lconn = task_conn.lock().await;
-                write_state(&lconn, ttt_towrite)
-                    .unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e));
+                    write_state(&lconn, ttt_towrite)?;
                     drop(lconn);
                 }
-        });
-
-        // push each task into the vec task
-        mtask.push(task);
-    }
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
 
     // launch and wait for every handle
-    for handle in mtask {
-        handle.await.unwrap();
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {e}"),
+            Err(e) => eprintln!("Error with thread: {e}"),
+            _ => (),
+        }
+    }
+}
+
+/// Copies the Twitter profile into Mastodon
+#[tokio::main]
+pub async fn profile(config: Config, bot: Option<bool>) {
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            let token = get_oauth2_token(&config.twitter);
+
+            spawn(async move {
+                // get the user of the last tweet of the feed
+                let twitter_user =
+                    get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
+                        .await?
+                        .first()
+                        .ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
+                        .clone()
+                        .user
+                        .ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
+
+                let note = get_note_from_description(
+                    &twitter_user.description,
+                    &twitter_user.entities.description.urls,
+                );
+
+                let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
+
+                let display_name = Some(String::from_utf16_lossy(
+                    &twitter_user
+                        .name
+                        .encode_utf16()
+                        .take(30)
+                        .collect::<Vec<u16>>(),
+                ));
+
+                let header = match twitter_user.profile_banner_url {
+                    Some(h) => Some(base64_media(&h).await?),
+                    None => None,
+                };
+
+                let update_creds = UpdateCredentialsInputOptions {
+                    bot,
+                    display_name,
+                    note,
+                    avatar: Some(
+                        base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
+                            .await?,
+                    ),
+                    header,
+                    fields_attributes,
+                    ..Default::default()
+                };
+
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                mastodon.update_credentials(Some(&update_creds)).await?;
+
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {e}"),
+            Err(e) => eprintln!("Error with thread: {e}"),
+            _ => (),
+        }
     }
 }

src/main.rs

@@ -1,5 +1,5 @@
-use clap::{App, Arg, SubCommand};
-use log::{error, LevelFilter};
+use clap::{Arg, ArgAction, Command};
+use log::LevelFilter;
 use scootaloo::*;
 use simple_logger::SimpleLogger;
 use std::str::FromStr;
@@ -7,136 +7,177 @@ use std::str::FromStr;
 const DEFAULT_CONFIG_PATH: &str = "/usr/local/etc/scootaloo.toml";
 
 fn main() {
-    let matches = App::new(env!("CARGO_PKG_NAME"))
+    let matches = Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))
         .about("A Twitter to Mastodon bot")
         .arg(
-            Arg::with_name("config")
-                .short("c")
+            Arg::new("config")
+                .short('c')
                 .long("config")
                 .value_name("CONFIG_FILE")
-                .help(&format!(
-                    "TOML config file for scootaloo (default {})",
-                    DEFAULT_CONFIG_PATH
-                ))
-                .takes_value(true)
+                .help("TOML config file for scootaloo")
+                .num_args(1)
+                .default_value(DEFAULT_CONFIG_PATH)
                 .display_order(1),
         )
         .arg(
-            Arg::with_name("log_level")
-                .short("l")
+            Arg::new("log_level")
+                .short('l')
                 .long("loglevel")
                 .value_name("LOGLEVEL")
-                .help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
-                .takes_value(true)
+                .help("Log level")
+                .num_args(1)
+                .value_parser(["Off", "Warn", "Error", "Info", "Debug"])
                 .display_order(2),
         )
         .subcommand(
-            SubCommand::with_name("register")
+            Command::new("register")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to register to a Mastodon Instance")
                 .arg(
-                    Arg::with_name("host")
-                        .short("H")
+                    Arg::new("host")
+                        .short('H')
                         .long("host")
                         .value_name("HOST")
                         .help("Base URL of the Mastodon instance to register to (no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .required(true)
                         .display_order(1)
                     )
                 .arg(
-                    Arg::with_name("name")
-                        .short("n")
+                    Arg::new("name")
+                        .short('n')
                         .long("name")
                         .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .required(true)
                         .display_order(2)
                 ),
         )
         .subcommand(
-            SubCommand::with_name("init")
+            Command::new("init")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to init Scootaloo DB")
                 .arg(
-                    Arg::with_name("config")
-                        .short("c")
+                    Arg::new("config")
+                        .short('c')
                         .long("config")
                         .value_name("CONFIG_FILE")
-                        .help(&format!(
-                            "TOML config file for scootaloo (default {})",
-                            DEFAULT_CONFIG_PATH
-                        ))
-                        .takes_value(true)
+                        .help("TOML config file for scootaloo")
+                        .default_value(DEFAULT_CONFIG_PATH)
+                        .num_args(1)
                         .display_order(1),
                 ),
         )
         .subcommand(
-            SubCommand::with_name("migrate")
+            Command::new("migrate")
             .version(env!("CARGO_PKG_VERSION"))
             .about("Command to migrate Scootaloo DB")
             .arg(
-                Arg::with_name("config")
-                    .short("c")
+                Arg::new("config")
+                    .short('c')
                     .long("config")
                     .value_name("CONFIG_FILE")
-                    .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
-                    .takes_value(true)
+                    .help("TOML config file for scootaloo")
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
                     .display_order(1),
             )
             .arg(
-                    Arg::with_name("name")
-                        .short("n")
+                Arg::new("name")
+                    .short('n')
                     .long("name")
                     .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .takes_value(true)
+                    .num_args(1)
                     .display_order(2)
             )
         )
+        .subcommand(
+            Command::new("copyprofile")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to copy a Twitter profile into Mastodon")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help("TOML config file for scootaloo")
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Mastodon Config name (as seen in the config file)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+            .arg(
+                Arg::new("bot")
+                    .short('b')
+                    .long("bot")
+                    .help("Declare user as bot")
+                    .action(ArgAction::SetTrue)
+                    .display_order(3)
+            )
+        )
         .get_matches();
 
     match matches.subcommand() {
-        ("register", Some(sub_m)) => {
+        Some(("register", sub_m)) => {
             register(
-                sub_m.value_of("host").unwrap(),
-                sub_m.value_of("name").unwrap(),
+                sub_m.get_one::<String>("host").unwrap(),
+                sub_m.get_one::<String>("name").unwrap(),
             );
             return;
         }
-        ("init", Some(sub_m)) => {
-            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+        Some(("init", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
             init_db(&config.scootaloo.db_path).unwrap();
             return;
         }
-        ("migrate", Some(sub_m)) => {
-            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+        Some(("migrate", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
             let config_twitter_screen_name =
                 &config.mastodon.values().next().unwrap().twitter_screen_name;
             migrate_db(
                 &config.scootaloo.db_path,
-                sub_m.value_of("name").unwrap_or(config_twitter_screen_name),
+                sub_m
+                    .get_one::<String>("name")
+                    .unwrap_or(config_twitter_screen_name),
             )
             .unwrap();
             return;
         }
+        Some(("copyprofile", sub_m)) => {
+            let mut config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            // filters out the user passed in cli from the global configuration
+            if let Some(m) = sub_m.get_one::<String>("name") {
+                match config.mastodon.get(m) {
+                    Some(_) => {
+                        config.mastodon.retain(|k, _| k == m);
+                    }
+                    None => panic!("Config file does not contain conf for {}", &m),
+                }
+            }
+
+            profile(config, sub_m.get_flag("bot").then_some(true));
+
+            return;
+        }
         _ => (),
     }
 
-    if matches.is_present("log_level") {
-        match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
-            Ok(level) => SimpleLogger::new().with_level(level).init().unwrap(),
-            Err(e) => {
+    if let Some(level) = matches.get_one::<String>("log_level") {
         SimpleLogger::new()
-                    .with_level(LevelFilter::Error)
+            .with_level(LevelFilter::from_str(level).unwrap())
             .init()
             .unwrap();
-                error!("Unknown log level filter: {}", e);
-            }
-        };
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+    let config = parse_toml(matches.get_one::<String>("config").unwrap());
 
     run(config);
 }

src/mastodon.rs

@@ -2,89 +2,189 @@ use crate::config::MastodonConfig;
 
 use egg_mode::{
     entities::{MentionEntity, UrlEntity},
-    tweet::Tweet,
+    user::UserEntityDetail,
 };
-use elefren::{apps::App, prelude::*, scopes::Scopes};
-use html_escape::decode_html_entities;
-use std::{borrow::Cow, collections::HashMap, io::stdin};
+use megalodon::{
+    generator,
+    mastodon::Mastodon,
+    megalodon::{AppInputOptions, CredentialsFieldAttribute},
+};
+use regex::Regex;
+use std::{collections::HashMap, io::stdin};
 
 /// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse
-fn twitter_mentions(ums: &[MentionEntity]) -> HashMap<String, String> {
-    ums.iter()
+/// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
+/// as well
+pub fn twitter_mentions(
+    toot: &mut String,
+    ums: &[MentionEntity],
+    masto: &HashMap<String, MastodonConfig>,
+) {
+    let tm: HashMap<String, String> = ums
+        .iter()
         .map(|s| {
             (
                 format!("@{}", s.screen_name),
                 format!("@{}@twitter.com", s.screen_name),
             )
         })
-        .collect()
+        .chain(
+            masto
+                .values()
+                .filter(|s| s.mastodon_screen_name.is_some())
+                .map(|s| {
+                    (
+                        format!("@{}", s.twitter_screen_name),
+                        format!(
+                            "@{}@{}",
+                            s.mastodon_screen_name.as_ref().unwrap(),
+                            s.base.split('/').last().unwrap()
+                        ),
+                    )
+                })
+                .collect::<HashMap<String, String>>(),
+        )
+        .collect();
+
+    for (k, v) in tm {
+        *toot = toot.replace(&k, &v);
+    }
 }
 
-/// Decodes urls from UrlEntities
-fn decode_urls(urls: &[UrlEntity]) -> HashMap<String, String> {
+/// Decodes urls in toot
+pub fn decode_urls(toot: &mut String, urls: &HashMap<String, String>) {
+    for (k, v) in urls {
+        *toot = toot.replace(k, v);
+    }
+}
+
+/// Reassociates source url with destination url for rewritting
+/// this takes a Teet UrlEntity and an optional Regex
+pub fn associate_urls(urls: &[UrlEntity], re: &Option<Regex>) -> HashMap<String, String> {
     urls.iter()
         .filter(|s| s.expanded_url.is_some())
         .map(|s| {
-            (
-                s.url.to_owned(),
-                s.expanded_url.as_deref().unwrap().to_owned(),
-            )
+            (s.url.to_owned(), {
+                let mut def = s.expanded_url.as_deref().unwrap().to_owned();
+
+                if let Some(r) = re {
+                    if r.is_match(s.expanded_url.as_deref().unwrap()) {
+                        def = s.display_url.to_owned();
+                    }
+                }
+
+                def
             })
-        .collect()
+        })
+        .collect::<HashMap<String, String>>()
+}
+
+/// Replaces the commonly used services by mirrors, if asked to
+pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<String, String>) {
+    for val in urls.values_mut() {
+        for (k, v) in alts {
+            *val = val.replace(&format!("/{k}/"), &format!("/{v}/"));
+        }
+    }
+}
+
+/// Finds a Mastodon screen_name/base_url from a MastodonConfig
+pub fn find_mastodon_screen_name_by_twitter_screen_name(
+    twitter_screen_name: &str,
+    masto: &HashMap<String, MastodonConfig>,
+) -> Option<(String, String)> {
+    masto.iter().find_map(|(_, v)| {
+        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
+            Some((
+                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
+                v.base.to_owned(),
+            ))
+        } else {
+            None
+        }
+    })
+}
+
+/// Replaces the original quoted tweet by the corresponding toot
+pub fn replace_tweet_by_toot(
+    urls: &mut HashMap<String, String>,
+    twitter_screen_name: &str,
+    tweet_id: u64,
+    mastodon_screen_name: &str,
+    base_url: &str,
+    toot_id: &str,
+) {
+    for val in urls.values_mut() {
+        if val.to_lowercase().starts_with(&format!(
+            "https://twitter.com/{}/status/{}",
+            twitter_screen_name.to_lowercase(),
+            tweet_id
+        )) {
+            *val = format!("{base_url}/@{mastodon_screen_name}/{toot_id}");
+        }
+    }
 }
 
 /// Gets Mastodon Data
 pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
-        base: Cow::from(masto.base.to_owned()),
-        client_id: Cow::from(masto.client_id.to_owned()),
-        client_secret: Cow::from(masto.client_secret.to_owned()),
-        redirect: Cow::from(masto.redirect.to_owned()),
-        token: Cow::from(masto.token.to_owned()),
-    };
-
-    Mastodon::from(data)
+    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
 }
 
-/// Builds toot text from tweet
-pub fn build_basic_status(tweet: &Tweet) -> String {
-    let mut toot = tweet.text.to_owned();
+/// Gets note from twitter_user description
+pub fn get_note_from_description(t: &Option<String>, urls: &[UrlEntity]) -> Option<String> {
+    t.as_ref().map(|d| {
+        let mut n = d.to_owned();
+        let a_urls = associate_urls(urls, &None);
+        decode_urls(&mut n, &a_urls);
+        n
+    })
+}
 
-    for decoded_url in decode_urls(&tweet.entities.urls) {
-        toot = toot.replace(&decoded_url.0, &decoded_url.1);
-    }
-
-    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions) {
-        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
-    }
-
-    decode_html_entities(&toot).to_string()
+/// Gets fields_attribute from UserEntityDetail
+pub fn get_attribute_from_url(
+    user_entity_detail: &Option<UserEntityDetail>,
+) -> Option<Vec<CredentialsFieldAttribute>> {
+    user_entity_detail.as_ref().and_then(|u| {
+        u.urls.first().and_then(|v| {
+            v.expanded_url.as_ref().map(|e| {
+                vec![CredentialsFieldAttribute {
+                    name: v.display_url.to_string(),
+                    value: e.to_string(),
+                }]
+            })
+        })
+    })
 }
 
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str, screen_name: &str) {
-    let mut builder = App::builder();
-    builder
-        .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
-        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
-        .scopes(Scopes::write_all())
-        .website(Cow::from(
-            "https://framagit.org/veretcle/scootaloo".to_string(),
-        ));
+#[tokio::main]
+pub async fn register(host: &str, screen_name: &str) {
+    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
 
-    let app = builder.build().expect("Cannot build the app");
+    let options = AppInputOptions {
+        redirect_uris: None,
+        scopes: Some(
+            [
+                "read:accounts".to_string(),
+                "write:accounts".to_string(),
+                "write:media".to_string(),
+                "write:statuses".to_string(),
+            ]
+            .to_vec(),
+        ),
+        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
+    };
 
-    let registration = Registration::new(host)
-        .register(app)
-        .expect("Cannot build registration object");
-    let url = registration
-        .authorize_url()
-        .expect("Cannot generate registration URI!");
+    let app_data = mastodon
+        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
+        .await
+        .expect("Cannot build registration object!");
 
-    println!("Click this link to authorize on Mastodon: {}", url);
+    let url = app_data.url.expect("Cannot generate registration URI!");
+
+    println!("Click this link to authorize on Mastodon: {url}");
     println!("Paste the returned authorization code: ");
 
     let mut input = String::new();
@@ -92,21 +192,47 @@ pub fn register(host: &str, screen_name: &str) {
         .read_line(&mut input)
         .expect("Unable to read back registration code!");
 
-    let code = input.trim();
-    let mastodon = registration
-        .complete(code)
+    let token_data = mastodon
+        .fetch_access_token(
+            app_data.client_id.to_owned(),
+            app_data.client_secret.to_owned(),
+            input.trim().to_string(),
+            megalodon::default::NO_REDIRECT.to_string(),
+        )
+        .await
         .expect("Unable to create access token!");
 
-    let toml = toml::to_string(&*mastodon).unwrap();
+    let mastodon = generator(
+        megalodon::SNS::Mastodon,
+        host.to_string(),
+        Some(token_data.access_token.to_owned()),
+        None,
+    );
+
+    let current_account = mastodon
+        .verify_account_credentials()
+        .await
+        .expect("Unable to access account information!")
+        .json();
 
     println!(
-        "Please insert the following block at the end of your configuration file:
+        r#"Please insert the following block at the end of your configuration file:
 [mastodon.{}]
-twitter_screen_name = \"{}\"
-{}",
+twitter_screen_name = "{}"
+mastodon_screen_name = "{}"
+base = "{}"
+client_id = "{}"
+client_secret = "{}"
+redirect = "{}"
+token = "{}""#,
         screen_name.to_lowercase(),
         screen_name,
-        toml
+        current_account.username,
+        host,
+        app_data.client_id,
+        app_data.client_secret,
+        app_data.redirect_uri,
+        token_data.access_token,
     );
 }
 
@@ -114,115 +240,343 @@ twitter_screen_name = \"{}\"
 mod tests {
     use super::*;
 
-    use chrono::prelude::*;
-    use egg_mode::tweet::TweetEntities;
-
     #[test]
-    fn test_twitter_mentions() {
-        let mention_entity = MentionEntity {
-            id: 12345,
-            range: (1, 3),
-            name: "Ta Mere l0l".to_string(),
-            screen_name: "tamerelol".to_string(),
+    fn test_get_attribute_from_url() {
+        let expected_credentials_field_attribute = CredentialsFieldAttribute {
+            name: "Nintendojo.fr".to_string(),
+            value: "https://www.nintendojo.fr".to_string(),
         };
 
-        let twitter_ums = vec![mention_entity];
+        let true_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr".to_string()),
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
 
-        let mut expected_mentions = HashMap::new();
-        expected_mentions.insert(
-            "@tamerelol".to_string(),
-            "@tamerelol@twitter.com".to_string(),
+        let false_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: None,
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        assert!(get_attribute_from_url(&None).is_none());
+        assert!(get_attribute_from_url(&Some(UserEntityDetail { urls: false_urls })).is_none());
+
+        let binding = get_attribute_from_url(&Some(UserEntityDetail { urls: true_urls })).unwrap();
+        let result_credentials_field_attribute = binding.first().unwrap();
+
+        assert_eq!(
+            result_credentials_field_attribute.name,
+            expected_credentials_field_attribute.name
+        );
+        assert_eq!(
+            result_credentials_field_attribute.value,
+            expected_credentials_field_attribute.value
         );
-
-        let decoded_mentions = twitter_mentions(&twitter_ums);
-
-        assert_eq!(expected_mentions, decoded_mentions);
     }
 
     #[test]
-    fn test_decode_urls() {
-        let url_entity1 = UrlEntity {
+    fn test_get_note_from_description() {
+        let urls = vec![UrlEntity {
             display_url: "tamerelol".to_string(),
             expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
             range: (1, 3),
             url: "https://t.me/tamerelol".to_string(),
-        };
+        }];
 
-        let url_entity2 = UrlEntity {
-            display_url: "tamerelol".to_string(),
-            expanded_url: None,
-            range: (1, 3),
-            url: "https://t.me/tamerelol".to_string(),
-        };
+        let some_description = Some("Youpi | https://t.me/tamerelol".to_string());
+        let none_description = None;
 
-        let twitter_urls = vec![url_entity1, url_entity2];
-
-        let mut expected_urls = HashMap::new();
-        expected_urls.insert(
-            "https://t.me/tamerelol".to_string(),
-            "https://www.nintendojo.fr/dojobar".to_string(),
+        assert_eq!(
+            get_note_from_description(&some_description, &urls),
+            Some("Youpi | https://www.nintendojo.fr/dojobar".to_string())
         );
 
-        let decoded_urls = decode_urls(&twitter_urls);
-
-        assert_eq!(expected_urls, decoded_urls);
+        assert_eq!(get_note_from_description(&none_description, &urls), None);
     }
 
     #[test]
-    fn test_build_basic_status() {
-        let t = Tweet {
-            coordinates: None,
-            created_at: Utc::now(),
-            current_user_retweet: None,
-            display_text_range: None,
-            entities: TweetEntities {
-                hashtags: vec![],
-                symbols: vec![],
-                urls: vec![
-                        UrlEntity {
-                            display_url: "youtube.com/watch?v=w5TrSa…".to_string(),
-                            expanded_url: Some("https://www.youtube.com/watch?v=w5TrSaoYmZ8".to_string()),
-                            range: (93, 116),
-                            url: "https://t.co/zXw0FfX2Nt".to_string(),
+    fn test_replace_tweet_by_toot() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
+            ),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+        ]);
+
+        replace_tweet_by_toot(
+            &mut associated_urls,
+            "NintendojoFR",
+            1590047921633755136,
+            "nintendojofr",
+            "https://m.nintendojo.fr",
+            "109309605486908797",
+        );
+
+        assert_eq!(associated_urls, expected_urls);
     }
-                ],
-                user_mentions: vec![
-                        MentionEntity {
-                            id: 491500016,
-                            range: (80, 95),
-                            name: "Nintendo France".to_string(),
-                            screen_name: "NintendoFrance".to_string(),
-                        }
-                    ],
-                media: None,
+
+    #[test]
+    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
+        let masto_config = HashMap::from([
+            (
+                "test".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tonpere".to_string(),
+                    mastodon_screen_name: Some("lalali".to_string()),
+                    twitter_page_size: None,
+                    base: "https://mstdn.net".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
                 },
-            extended_entities: None,
-            favorite_count: 0,
-            favorited: None,
-            filter_level: None,
-            id: 1491541246984306693,
-            in_reply_to_user_id: None,
-            in_reply_to_screen_name: None,
-            in_reply_to_status_id: None,
-            lang: None,
-            place: None,
-            possibly_sensitive: None,
-            quoted_status: None,
-            quoted_status_id: None,
-            retweet_count: 0,
-            retweeted: None,
-            retweeted_status: None,
-            source: None,
-            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance https://t.co/zXw0FfX2Nt".to_string(),
-            truncated: false,
-            user: None,
-            withheld_copyright: false,
-            withheld_in_countries: None,
-            withheld_scope: None,
-        };
+            ),
+            (
+                "test2".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tamerelol".to_string(),
+                    mastodon_screen_name: None,
+                    twitter_page_size: None,
+                    base: "https://mastoot.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test3".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "NintendojoFR".to_string(),
+                    mastodon_screen_name: Some("nintendojofr".to_string()),
+                    twitter_page_size: None,
+                    base: "https://m.nintendojo.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+        ]);
 
-        let t_out = build_basic_status(&t);
+        // case sensitiveness, to avoid any mistake
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
+        );
+        assert_eq!(
+            Some((
+                "nintendojofr".to_string(),
+                "https://m.nintendojo.fr".to_string()
+            )),
+            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
+        );
+        // should return None if twitter_screen_name is undefined
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
+        );
+        assert_eq!(
+            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
+            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
+        );
+    }
 
-        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com https://www.youtube.com/watch?v=w5TrSaoYmZ8");
+    #[test]
+    fn test_twitter_mentions() {
+        let mention_entities = vec![
+            MentionEntity {
+                id: 12345,
+                range: (1, 3),
+                name: "Ta Mere l0l".to_string(),
+                screen_name: "tamerelol".to_string(),
+            },
+            MentionEntity {
+                id: 6789,
+                range: (1, 3),
+                name: "TONPERE".to_string(),
+                screen_name: "tonpere".to_string(),
+            },
+        ];
+
+        let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
+
+        let masto_config = HashMap::from([(
+            "test".to_string(),
+            (MastodonConfig {
+                twitter_screen_name: "tonpere".to_string(),
+                mastodon_screen_name: Some("lalali".to_string()),
+                twitter_page_size: None,
+                base: "https://mstdn.net".to_string(),
+                client_id: "".to_string(),
+                client_secret: "".to_string(),
+                redirect: "".to_string(),
+                token: "".to_string(),
+            }),
+        )]);
+
+        twitter_mentions(&mut toot, &mention_entities, &masto_config);
+
+        assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
+    }
+
+    #[test]
+    fn test_decode_urls() {
+        let urls = HashMap::from([
+            (
+                "https://t.co/thisisatest".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/nopenotinclusive".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
+
+        let mut toot =
+            "Rendez-vous sur https://t.co/thisisatest avec nos https://t.co/nopenotinclusive !"
+                .to_string();
+
+        decode_urls(&mut toot, &urls);
+
+        assert_eq!(
+            &toot,
+            "Rendez-vous sur https://www.nintendojo.fr/dojobar avec nos invité.es !"
+        );
+    }
+
+    #[test]
+    fn test_associate_urls() {
+        let urls = vec![
+            UrlEntity {
+                display_url: "tamerelol".to_string(),
+                expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+                range: (1, 3),
+                url: "https://t.me/tamerelol".to_string(),
+            },
+            UrlEntity {
+                display_url: "sadcat".to_string(),
+                expanded_url: None,
+                range: (1, 3),
+                url: "https://t.me/sadcat".to_string(),
+            },
+            UrlEntity {
+                display_url: "invité.es".to_string(),
+                expanded_url: Some("http://xn--invit-fsa.es".to_string()),
+                range: (85, 108),
+                url: "https://t.co/WAUgnpHLmo".to_string(),
+            },
+        ];
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.me/tamerelol".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/WAUgnpHLmo".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
+
+        let re = Regex::new("(.+)\\.es$").ok();
+
+        let associated_urls = associate_urls(&urls, &re);
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_replace_alt_services() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://www.youtube.com/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://twitter.com/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        let alt_services = HashMap::from([
+            ("twitter.com".to_string(), "nitter.net".to_string()),
+            ("youtu.be".to_string(), "invidio.us".to_string()),
+            ("www.youtube.com".to_string(), "invidio.us".to_string()),
+            ("youtube.com".to_string(), "invidio.us".to_string()),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://invidio.us/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://nitter.net/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        replace_alt_services(&mut associated_urls, &alt_services);
+
+        assert_eq!(associated_urls, expected_urls);
     }
 }

src/state.rs

@@ -21,8 +21,8 @@ pub fn read_state(
 ) -> Result<Option<TweetToToot>, Box<dyn Error>> {
     debug!("Reading tweet_id {:?}", s);
     let query: String = match s {
-        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {} and twitter_screen_name = \"{}\"", i, n),
-        None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{}\" ORDER BY tweet_id DESC LIMIT 1", n),
+        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {i} and twitter_screen_name = \"{n}\""),
+        None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{n}\" ORDER BY tweet_id DESC LIMIT 1"),
     };
 
     let mut stmt = conn.prepare(&query)?;
@@ -78,8 +78,7 @@ pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
         &format!(
             "ALTER TABLE tweet_to_toot
         ADD COLUMN twitter_screen_name TEXT NOT NULL
-        DEFAULT \"{}\"",
-            s
+        DEFAULT \"{s}\""
         ),
         [],
     );
@@ -87,7 +86,7 @@ pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
     match res {
         Err(e) => match e.to_string().as_str() {
             "duplicate column name: twitter_screen_name" => Ok(()),
-            _ => Err(Box::new(e)),
+            _ => Err(e.into()),
         },
         _ => Ok(()),
     }

src/twitter.rs

@@ -27,19 +27,31 @@ pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
     }
 }
 
-/// Gets Twitter user timeline
+/// Gets Twitter user timeline, eliminate responses to others and reverse it
 pub async fn get_user_timeline(
     screen_name: &str,
     token: &Token,
     lid: Option<u64>,
+    page_size: i32,
 ) -> Result<Vec<Tweet>, Box<dyn Error>> {
     // fix the page size to 200 as it is the maximum Twitter authorizes
     let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
-        .with_page_size(200)
+        .with_page_size(page_size)
         .older(lid)
         .await?;
 
-    Ok(feed.to_vec())
+    let mut feed: Vec<Tweet> = feed
+        .iter()
+        .cloned()
+        .filter(|t| match &t.in_reply_to_screen_name {
+            Some(r) => r.to_lowercase() == screen_name.to_lowercase(),
+            None => true,
+        })
+        .collect();
+
+    feed.reverse();
+
+    Ok(feed)
 }
 
 /// Retrieves a single media from a tweet and store it in a temporary file

src/util.rs

@@ -1,18 +1,21 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
 
-use std::{borrow::Cow, error::Error};
-
+use base64::encode;
 use egg_mode::tweet::Tweet;
-
-use elefren::prelude::*;
-
+use futures::{stream, stream::StreamExt};
 use log::{error, info, warn};
-
+use megalodon::{
+    entities::UploadMedia::{AsyncAttachment, Attachment},
+    error,
+    mastodon::Mastodon,
+    megalodon::Megalodon,
+};
 use reqwest::Url;
-
+use std::error::Error;
 use tokio::{
     fs::{create_dir_all, remove_file, File},
     io::copy,
+    time::{sleep, Duration},
 };
 
 /// Generate associative table between media ids and tweet extended entities
@@ -25,63 +28,48 @@ pub async fn generate_media_ids(
     let mut media_ids: Vec<String> = vec![];
 
     if let Some(m) = &tweet.extended_entities {
-        // create tasks list
-        let mut tasks = vec![];
-
-        // size of media_ids vector, should be equal to the media vector
-        media_ids.resize(m.media.len(), String::new());
-
         info!("{} medias in tweet", m.media.len());
 
-        for (i, media) in m.media.iter().enumerate() {
+        let medias = m.media.clone();
+
+        let mut stream = stream::iter(medias)
+            .map(|media| {
                 // attribute media url
                 media_url = media.url.clone();
 
                 // clone everything we need
                 let cache_path = String::from(cache_path);
-            let media = media.clone();
                 let mastodon = mastodon.clone();
 
-            let task = tokio::task::spawn(async move {
+                tokio::task::spawn(async move {
                     info!("Start treating {}", media.media_url_https);
                     // get the tweet embedded media
-                let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
-                    Ok(l) => l,
-                    Err(e) => {
-                        return Err(ScootalooError::new(&format!(
-                            "Cannot get tweet media for {}: {}",
-                            &media.url, e
-                        )))
-                    }
-                };
+                    let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
 
                     // upload media to Mastodon
-                let mastodon_media = mastodon.media(Cow::from(local_tweet_media_path.to_owned()));
+                    let mastodon_media = mastodon
+                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        .await?
+                        .json();
                     // at this point, we can safely erase the original file
                     // it doesn’t matter if we can’t remove, cache_media fn is idempotent
                     remove_file(&local_tweet_media_path).await.ok();
 
-                let mastodon_media = match mastodon_media {
-                    Ok(m) => m,
-                    Err(e) => {
-                        return Err(ScootalooError::new(&format!(
-                            "Attachment {} cannot be uploaded to Mastodon Instance: {}",
-                            &local_tweet_media_path, e
-                        )))
-                    }
+                    let id = match mastodon_media {
+                        Attachment(m) => m.id,
+                        AsyncAttachment(m) => wait_until_uploaded(&mastodon, &m.id).await?,
                     };
 
-                Ok((i, mastodon_media.id))
-            });
+                    Ok::<String, ScootalooError>(id)
+                })
+            })
+            .buffered(4); // there are max four medias per tweet and they need to be treated in
+                          // order
 
-            tasks.push(task);
-        }
-
-        for task in tasks {
-            match task.await {
-                // insert the media at the right place
-                Ok(Ok((i, v))) => media_ids[i] = v,
-                Ok(Err(e)) => warn!("{}", e),
+        while let Some(result) = stream.next().await {
+            match result {
+                Ok(Ok(v)) => media_ids.push(v),
+                Ok(Err(e)) => warn!("Cannot treat media: {}", e),
                 Err(e) => error!("Something went wrong when joining the main thread: {}", e),
             }
         }
@@ -95,6 +83,45 @@ pub async fn generate_media_ids(
     (media_url, media_ids)
 }
 
+/// Wait on uploaded medias when necessary
+async fn wait_until_uploaded(client: &Mastodon, id: &str) -> Result<String, error::Error> {
+    loop {
+        sleep(Duration::from_secs(1)).await;
+        let res = client.get_media(id.to_string()).await;
+        return match res {
+            Ok(res) => Ok(res.json.id),
+            Err(err) => match err {
+                error::Error::OwnError(ref own_err) => match own_err.kind {
+                    error::Kind::HTTPPartialContentError => continue,
+                    _ => Err(err),
+                },
+                _ => Err(err),
+            },
+        };
+    }
+}
+
+/// Transforms the media into a base64 equivalent
+pub async fn base64_media(u: &str) -> Result<String, Box<dyn Error>> {
+    let mut response = reqwest::get(u).await?;
+
+    let mut buffer = Vec::new();
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut buffer).await?;
+    }
+
+    let content_type = response
+        .headers()
+        .get("content-type")
+        .ok_or_else(|| ScootalooError::new(&format!("Cannot get media content type for {u}")))?
+        .to_str()?;
+
+    let encoded_f = encode(buffer);
+
+    Ok(format!("data:{content_type};base64,{encoded_f}"))
+}
+
 /// Gets and caches Twitter Media inside the determined temp dir
 pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
     // create dir
@@ -109,19 +136,17 @@ pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
         .path_segments()
         .ok_or_else(|| {
             ScootalooError::new(&format!(
-                "Cannot determine the destination filename for {}",
-                u
+                "Cannot determine the destination filename for {u}"
             ))
         })?
         .last()
         .ok_or_else(|| {
             ScootalooError::new(&format!(
-                "Cannot determine the destination filename for {}",
-                u
+                "Cannot determine the destination filename for {u}"
             ))
         })?;
 
-    let dest_filepath = format!("{}/{}", t, dest_filename);
+    let dest_filepath = format!("{t}/{dest_filename}");
 
     let mut dest_file = File::create(&dest_filepath).await?;
 
@@ -153,4 +178,17 @@ mod tests {
 
         remove_dir_all(TMP_DIR).unwrap();
     }
+
+    #[tokio::test]
+    async fn test_base64_media() {
+        let img = base64_media(
+            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
+        )
+        .await
+        .unwrap();
+
+        assert!(img.starts_with("data:image/png;base64,"));
+
+        assert!(img.ends_with("="));
+    }
 }

tests/config.rs

@@ -1,4 +1,108 @@
 use scootaloo::parse_toml;
+use std::collections::HashMap;
+
+#[test]
+fn test_alt_services() {
+    let toml = parse_toml("tests/no_test_alt_services.toml");
+    assert_eq!(toml.scootaloo.alternative_services_for, None);
+
+    let toml = parse_toml("tests/test_alt_services.toml");
+    assert_eq!(
+        toml.scootaloo.alternative_services_for,
+        Some(HashMap::from([
+            ("tamere.lol".to_string(), "tonpere.mdr".to_string()),
+            ("you.pi".to_string(), "you.pla".to_string())
+        ]))
+    );
+}
+
+#[test]
+fn test_re_display() {
+    let toml = parse_toml("tests/no_show_url_as_display_url_for.toml");
+    assert_eq!(toml.scootaloo.show_url_as_display_url_for, None);
+
+    let toml = parse_toml("tests/show_url_as_display_url_for.toml");
+
+    assert_eq!(
+        toml.scootaloo.show_url_as_display_url_for,
+        Some("^(.+)\\.es$".to_string())
+    );
+}
+
+#[test]
+fn test_page_size() {
+    const DEFAULT_PAGE_SIZE: i32 = 200;
+    let toml = parse_toml("tests/page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, Some(100));
+
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42));
+
+    // this is the exact line that is used inside fn run() to determine the twitter page size
+    // passed to fn get_user_timeline()
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+    let page_size_for_1 = toml
+        .mastodon
+        .get("1")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 100);
+    assert_eq!(page_size_for_1, 42);
+
+    let toml = parse_toml("tests/no_page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, None);
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    // and same here
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 200);
+}
+
+#[test]
+fn test_parse_good_toml_rate_limit() {
+    let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml");
+
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize));
+}
+
+#[test]
+fn test_parse_good_toml_mastodon_screen_name() {
+    let parse_good_toml = parse_toml("tests/good_test_mastodon_screen_name.toml");
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("0")
+            .unwrap()
+            .mastodon_screen_name,
+        Some("tarace".to_string())
+    );
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("1")
+            .unwrap()
+            .mastodon_screen_name,
+        None
+    );
+}
 
 #[test]
 fn test_parse_good_toml() {
@@ -9,6 +113,7 @@ fn test_parse_good_toml() {
         "/var/random/scootaloo.sqlite"
     );
     assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, None);
 
     assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
     assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");

tests/good_test_mastodon_screen_name.toml

@@ -0,0 +1,28 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+mastodon_screen_name="tarace"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/good_test_rate_limit.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+rate_limit=69
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_page_size.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_show_url_as_display_url_for.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_test_alt_services.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/page_size.toml

@@ -0,0 +1,29 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+page_size=100
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tonperemdr"
+twitter_page_size=42
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/show_url_as_display_url_for.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+show_url_as_display_url_for = "^(.+)\\.es$"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/test_alt_services.toml

@@ -0,0 +1,22 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr"
+"you.pi" = "you.pla"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"