Merge branch '12-remove-elefren-deprecated-in-favor-of-megalodon-rs' into 'master'

Remove elefren (deprecated) in favor of megalodon-rs

Closes #12

See merge request veretcle/scootaloo!47

Cargo.toml

@@ -1,21 +1,22 @@
 [package]
 name = "scootaloo"
-version = "0.6.1"
+version = "1.0.0"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-chrono = "^0.4"
+regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^2.34"
-futures = "^0.3"
+clap = "^4"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-tokio = { version = "1", features = ["full"]}
-elefren = "^0.22"
+isolang = "^2"
+tokio = { version = "^1", features = ["rt"]}
+futures = "^0.3"
+megalodon = "^0.2"
 html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
@@ -24,3 +25,4 @@ mime = "^0.3"
 
 [profile.release]
 strip = true
+lto = true

README.md

@@ -10,23 +10,38 @@ If any of the last steps failed, the Toot gets published with the exact same tex
 RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
 
 # Usage
+## Configuring
 
 First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this:
 
 ```toml
 [scootaloo]
-
-db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
-cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
+cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneously
+## optional, this should be omitted the majority of the time
+## sometimes, twitter try to use french inclusive writting, but instead of using `·` (median point), they’re using `.`
+## this makes twitter interpret it as a URL, which is wrong
+## this parameter allows you to catch such URLs and apply the `display_url` (i.e. `tout.es`) instead of the `expanded_url` (i.e. `http://tout.es`)
+## in those particular cases
+## (!) use with caution, it might have some undesired effects
+show_url_as_display_url_for = "^http(s)://(.+)\\.es$"
+## optional, this allows you to replace the host for popular services such as YouTube of Twitter, or any other
+## with their more freely accessible equivalent
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr" ## quotes are necessary for both parameters
+"you.pi" = "you.pla"
+"www.you.pi" = "you.pla" ## this is an exact match, so you’ll need to lay out all the possibilities
 
 [twitter]
-username="NintendojoFR" ## User Timeline to copy
-
 ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
-consumer_key="MYCONSUMERKEY"
-consumer_secret="MYCONSUMERSECRET"
-access_key="MYACCESSKEY"
-access_secret="MYACCESSSECRET"
+page_size = 20 ## optional, default 200, max number of tweet retrieved
+consumer_key = "MYCONSUMERKEY"
+consumer_secret = "MYCONSUMERSECRET"
+access_key = "MYACCESSKEY"
+access_secret = "MYACCESSSECRET"
+
+[mastodon]
 ```
 
 Then run the command with the `init` subcommand to initiate the DB:
@@ -44,7 +59,9 @@ scootaloo register --host https://m.nintendojo.fr
 This will give you the end of the TOML file. It will look like this:
 
 ```toml
-[mastodon]
+[mastodon.nintendojofr] ## account
+twitter_screen_name = "NintendojoFR" ## User Timeline to copy
+mastodon_screen_name = "nintendojofr" ## optional, Mastodon account name used for smart mentions
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
 client_secret = "MYCLIENTSECRET"
@@ -52,6 +69,15 @@ redirect = "urn:ietf:wg:oauth:2.0:oob"
 token = "MYTOKEN"
 ```
 
+You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
+
+You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so:
+```
+twitter_page_size = 40
+```
+
+## Running
+
 You can then run the application via `cron` for example. Here is the generic usage:
 
 ```sh
@@ -71,6 +97,7 @@ OPTIONS:
 SUBCOMMANDS:
     help        Prints this message or the help of the given subcommand(s)
     init        Command to init Scootaloo DB
+    migrate     Command to migrate Scootaloo DB
     register    Command to register to a Mastodon Instance
 ```
 
@@ -86,5 +113,17 @@ sqlite3 /var/lib/scootaloo/scootaloo.sqlite
 And inserting the data:
 
 ```sql
-INSERT INTO tweet_to_toot VALUES (1383782580412030982, "");
+INSERT INTO tweet_to_toot VALUES ("<twitter_screen_name>", 1383782580412030982, "<twitter_screen_name>");
 ```
+
+The last value is supposed to be the Toot ID. It cannot be null, so you better initialize it with something unique, like the Twitter Screen Name for example.
+
+# Migrating from Scootaloo ⩽ 0.6.1
+
+The DB scheme has change between version 0.6.x and 0.7.x (this is due to the multi-account nature of Scootaloo from 0.7.x onward). You need to migrate your DB. You can do so by issuing the command:
+
+```
+scootaloo migrate
+```
+
+You can optionnally specify a screen name with the `--name` option. By default, it’ll take the first screen name in the config file.

src/config.rs

@@ -1,25 +1,29 @@
+use std::{collections::HashMap, fs::read_to_string};
+
 use serde::Deserialize;
-use std::fs::read_to_string;
 
 /// General configuration Struct
 #[derive(Debug, Deserialize)]
 pub struct Config {
     pub twitter: TwitterConfig,
-    pub mastodon: MastodonConfig,
+    pub mastodon: HashMap<String, MastodonConfig>,
     pub scootaloo: ScootalooConfig,
 }
 
 #[derive(Debug, Deserialize)]
 pub struct TwitterConfig {
-    pub username: String,
     pub consumer_key: String,
     pub consumer_secret: String,
     pub access_key: String,
     pub access_secret: String,
+    pub page_size: Option<i32>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
 pub struct MastodonConfig {
+    pub twitter_screen_name: String,
+    pub mastodon_screen_name: Option<String>,
+    pub twitter_page_size: Option<i32>,
     pub base: String,
     pub client_id: String,
     pub client_secret: String,
@@ -31,6 +35,9 @@ pub struct MastodonConfig {
 pub struct ScootalooConfig {
     pub db_path: String,
     pub cache_path: String,
+    pub rate_limit: Option<usize>,
+    pub show_url_as_display_url_for: Option<String>,
+    pub alternative_services_for: Option<HashMap<String, String>>,
 }
 
 /// Parses the TOML file into a Config Struct

src/error.rs

@@ -1,8 +1,12 @@
 use std::{
+    boxed::Box,
+    convert::From,
     error::Error,
     fmt::{Display, Formatter, Result},
 };
 
+use megalodon::error::Error as megalodonError;
+
 #[derive(Debug)]
 pub struct ScootalooError {
     details: String,
@@ -23,3 +27,15 @@ impl Display for ScootalooError {
         write!(f, "{}", self.details)
     }
 }
+
+impl From<Box<dyn Error>> for ScootalooError {
+    fn from(error: Box<dyn Error>) -> Self {
+        ScootalooError::new(&format!("Error in a subset crate: {}", error))
+    }
+}
+
+impl From<megalodonError> for ScootalooError {
+    fn from(error: megalodonError) -> Self {
+        ScootalooError::new(&format!("Error in megalodon crate: {}", error))
+    }
+}

src/lib.rs

@@ -7,151 +7,238 @@ use config::Config;
 
 mod mastodon;
 pub use mastodon::register;
-use mastodon::{build_basic_status, get_mastodon_token};
+use mastodon::*;
 
 mod twitter;
 use twitter::*;
 
 mod util;
+use crate::util::generate_media_ids;
 
 mod state;
-pub use state::init_db;
+pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
-use elefren::{prelude::*, status_builder::StatusBuilder};
-use log::{debug, error, info, warn};
+use futures::StreamExt;
+use html_escape::decode_html_entities;
+use isolang::Language;
+use log::info;
+use megalodon::{megalodon::PostStatusInputOptions, Megalodon};
+use regex::Regex;
 use rusqlite::Connection;
-use std::borrow::Cow;
-use tokio::fs::remove_file;
+use std::sync::Arc;
+use tokio::{spawn, sync::Mutex};
+
+const DEFAULT_RATE_LIMIT: usize = 4;
+const DEFAULT_PAGE_SIZE: i32 = 200;
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
     // open the SQLite connection
-    let conn = Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
-        panic!(
-            "Something went wrong when opening the DB {}: {}",
-            &config.scootaloo.db_path, e
-        )
-    });
-    // retrieve the last tweet ID for the username
-    let last_tweet_id = read_state(&conn, None)
-        .unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
-        .map(|s| s.tweet_id);
-
-    // get OAuth2 token
-    let token = get_oauth2_token(&config.twitter);
-
-    // get Mastodon instance
-    let mastodon = get_mastodon_token(&config.mastodon);
-
-    // get user timeline feed (Vec<tweet>)
-    let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
-        .await
-        .unwrap_or_else(|e| {
+    let conn = Arc::new(Mutex::new(
+        Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
             panic!(
-                "Something went wrong when trying to retrieve {}’s timeline: {}",
-                &config.twitter.username, e
+                "Something went wrong when opening the DB {}: {}",
+                &config.scootaloo.db_path, e
             )
-        });
+        }),
+    ));
 
-    // empty feed -> exiting
-    if feed.is_empty() {
-        info!("Nothing to retrieve since last time, exiting…");
-        return;
-    }
+    let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
 
-    // order needs to be chronological
-    feed.reverse();
+    let display_url_re = config
+        .scootaloo
+        .show_url_as_display_url_for
+        .as_ref()
+        .map(|r|
+        // we want to panic in case the RE is not valid
+        Regex::new(r).unwrap());
 
-    for tweet in &feed {
-        debug!("Treating Tweet {} inside feed", tweet.id);
-        // initiate the toot_reply_id var
-        let mut toot_reply_id: Option<String> = None;
-        // determine if the tweet is part of a thread (response to self) or a standard response
-        if let Some(r) = &tweet.in_reply_to_screen_name {
-            if r.to_lowercase() != config.twitter.username.to_lowercase() {
-                // we are responding not threading
-                info!("Tweet is a direct response, skipping");
-                continue;
-            }
-            info!("Tweet is a thread");
-            toot_reply_id = read_state(&conn, tweet.in_reply_to_status_id)
-                .unwrap_or(None)
-                .map(|s| s.toot_id);
-        };
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            // calculate Twitter page size
+            let page_size = mastodon_config
+                .twitter_page_size
+                .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
 
-        // build basic status by just yielding text and dereferencing contained urls
-        let mut status_text = build_basic_status(tweet);
+            // create temporary value for each task
+            let scootaloo_cache_path = config.scootaloo.cache_path.clone();
+            let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
+            let display_url_re = display_url_re.clone();
+            let token = get_oauth2_token(&config.twitter);
+            let task_conn = conn.clone();
+            let global_mastodon_config = global_mastodon_config.clone();
 
-        let mut status_medias: Vec<String> = vec![];
-        // reupload the attachments if any
-        if let Some(m) = &tweet.extended_entities {
-            for media in &m.media {
-                let local_tweet_media_path =
-                    match get_tweet_media(media, &config.scootaloo.cache_path).await {
-                        Ok(m) => m,
-                        Err(e) => {
-                            error!("Cannot get tweet media for {}: {}", &media.url, e);
-                            continue;
+            spawn(async move {
+                info!("Starting treating {}", &mastodon_config.twitter_screen_name);
+                // retrieve the last tweet ID for the username
+                let lconn = task_conn.lock().await;
+                let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
+                    .map(|r| r.tweet_id);
+                drop(lconn);
+
+                // get reversed, curated user timeline
+                let feed = get_user_timeline(
+                    &mastodon_config.twitter_screen_name,
+                    &token,
+                    last_tweet_id,
+                    page_size,
+                )
+                .await?;
+
+                // get Mastodon instance
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                for tweet in &feed {
+                    info!("Treating Tweet {} inside feed", tweet.id);
+
+                    // basic toot text
+                    let mut status_text = tweet.text.clone();
+
+                    // add mentions and smart mentions
+                    if !&tweet.entities.user_mentions.is_empty() {
+                        info!("Tweet contains mentions, add them!");
+                        let global_mastodon_config = global_mastodon_config.lock().await;
+                        twitter_mentions(
+                            &mut status_text,
+                            &tweet.entities.user_mentions,
+                            &global_mastodon_config,
+                        );
+                        drop(global_mastodon_config);
+                    }
+
+                    if !&tweet.entities.urls.is_empty() {
+                        info!("Tweet contains links, add them!");
+                        let mut associated_urls =
+                            associate_urls(&tweet.entities.urls, &display_url_re);
+
+                        if let Some(q) = &tweet.quoted_status {
+                            if let Some(u) = &q.user {
+                                info!(
+                                    "Tweet {} contains a quote, we try to find it within the DB",
+                                    tweet.id
+                                );
+                                // we know we have a quote and a user, we can lock both the
+                                // connection to DB and global_config
+                                // we will release them manually as soon as they’re useless
+                                let lconn = task_conn.lock().await;
+                                let global_mastodon_config = global_mastodon_config.lock().await;
+                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
+                                {
+                                    info!("We have found the associated toot({})", &r.toot_id);
+                                    // drop conn immediately after the request: we won’t need it
+                                    // any more and the treatment there might be time-consuming
+                                    drop(lconn);
+                                    if let Some((m, t)) =
+                                        find_mastodon_screen_name_by_twitter_screen_name(
+                                            &r.twitter_screen_name,
+                                            &global_mastodon_config,
+                                        )
+                                    {
+                                        // drop the global conf, we have all we required, no need
+                                        // to block it further
+                                        drop(global_mastodon_config);
+                                        replace_tweet_by_toot(
+                                            &mut associated_urls,
+                                            &r.twitter_screen_name,
+                                            q.id,
+                                            &m,
+                                            &t,
+                                            &r.toot_id,
+                                        );
+                                    }
+                                }
+                            }
                         }
+
+                        if let Some(a) = &scootaloo_alt_services {
+                            replace_alt_services(&mut associated_urls, a);
+                        }
+
+                        decode_urls(&mut status_text, &associated_urls);
+                    }
+
+                    // building associative media list
+                    let (media_url, status_medias) =
+                        generate_media_ids(tweet, &scootaloo_cache_path, &mastodon).await;
+
+                    status_text = status_text.replace(&media_url, "");
+
+                    // now that the text won’t be altered anymore, we can safely remove HTML
+                    // entities
+                    status_text = decode_html_entities(&status_text).to_string();
+
+                    info!("Building corresponding Mastodon status");
+
+                    let mut post_status = PostStatusInputOptions {
+                        media_ids: None,
+                        poll: None,
+                        in_reply_to_id: None,
+                        sensitive: None,
+                        spoiler_text: None,
+                        visibility: None,
+                        scheduled_at: None,
+                        language: None,
+                        quote_id: None,
                     };
 
-                let mastodon_media_ids = match mastodon
-                    .media(Cow::from(local_tweet_media_path.to_owned()))
-                {
-                    Ok(m) => {
-                        remove_file(&local_tweet_media_path)
-                            .await
-                            .unwrap_or_else(|e|
-                                warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
-                            );
-                        m.id
+                    if !status_medias.is_empty() {
+                        post_status.media_ids = Some(status_medias);
                     }
-                    Err(e) => {
-                        error!(
-                            "Attachment {} cannot be uploaded to Mastodon Instance: {}",
-                            &local_tweet_media_path, e
-                        );
-                        continue;
+
+                    // thread if necessary
+                    if tweet.in_reply_to_user_id.is_some() {
+                        let lconn = task_conn.lock().await;
+                        if let Ok(Some(r)) = read_state(
+                            &lconn,
+                            &mastodon_config.twitter_screen_name,
+                            tweet.in_reply_to_status_id,
+                        ) {
+                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
+                        }
+                        drop(lconn);
                     }
-                };
 
-                status_medias.push(mastodon_media_ids);
+                    // language if any
+                    if let Some(l) = &tweet.lang {
+                        if let Some(r) = Language::from_639_1(l) {
+                            post_status.language = Some(r.to_string());
+                        }
+                    }
 
-                // last step, removing the reference to the media from with the toot’s text
-                status_text = status_text.replace(&media.url, "");
-            }
+                    // can be activated for test purposes
+                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
+
+                    let published_status = mastodon
+                        .post_status(status_text, Some(&post_status))
+                        .await?
+                        .json();
+                    // this will return if it cannot publish the status preventing the last_tweet from
+                    // being written into db
+
+                    let ttt_towrite = TweetToToot {
+                        twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
+                        tweet_id: tweet.id,
+                        toot_id: published_status.id,
+                    };
+
+                    // write the current state (tweet ID and toot ID) to avoid copying it another time
+                    let lconn = task_conn.lock().await;
+                    write_state(&lconn, ttt_towrite)?;
+                    drop(lconn);
+                }
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    // launch and wait for every handle
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {}", e),
+            Err(e) => eprintln!("Error with thread: {}", e),
+            _ => (),
         }
-        // finished reuploading attachments, now let’s do the toot baby!
-
-        debug!("Building corresponding Mastodon status");
-
-        let mut status_builder = StatusBuilder::new();
-
-        status_builder.status(&status_text).media_ids(status_medias);
-
-        if let Some(i) = toot_reply_id {
-            status_builder.in_reply_to(&i);
-        }
-
-        let status = status_builder
-            .build()
-            .unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
-
-        // publish status
-        // again unwrap is safe here as we are in the main thread
-        let published_status = mastodon.new_status(status).unwrap();
-        // this will panic if it cannot publish the status, which is a good thing, it allows the
-        // last_tweet gathered not to be written
-
-        let ttt_towrite = TweetToToot {
-            tweet_id: tweet.id,
-            toot_id: published_status.id,
-        };
-
-        // write the current state (tweet ID and toot ID) to avoid copying it another time
-        write_state(&conn, ttt_towrite)
-            .unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e));
     }
 }

src/main.rs

@@ -1,5 +1,5 @@
-use clap::{App, Arg, SubCommand};
-use log::{error, LevelFilter};
+use clap::{Arg, Command};
+use log::LevelFilter;
 use scootaloo::*;
 use simple_logger::SimpleLogger;
 use std::str::FromStr;
@@ -7,91 +7,133 @@ use std::str::FromStr;
 const DEFAULT_CONFIG_PATH: &str = "/usr/local/etc/scootaloo.toml";
 
 fn main() {
-    let matches = App::new(env!("CARGO_PKG_NAME"))
+    let matches = Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))
         .about("A Twitter to Mastodon bot")
         .arg(
-            Arg::with_name("config")
-                .short("c")
+            Arg::new("config")
+                .short('c')
                 .long("config")
                 .value_name("CONFIG_FILE")
-                .help(&format!(
-                    "TOML config file for scootaloo (default {})",
-                    DEFAULT_CONFIG_PATH
-                ))
-                .takes_value(true)
+                .help("TOML config file for scootaloo")
+                .num_args(1)
+                .default_value(DEFAULT_CONFIG_PATH)
                 .display_order(1),
         )
         .arg(
-            Arg::with_name("log_level")
-                .short("l")
+            Arg::new("log_level")
+                .short('l')
                 .long("loglevel")
                 .value_name("LOGLEVEL")
-                .help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
-                .takes_value(true)
+                .help("Log level")
+                .num_args(1)
+                .value_parser(["Off", "Warn", "Error", "Info", "Debug"])
                 .display_order(2),
         )
         .subcommand(
-            SubCommand::with_name("register")
+            Command::new("register")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to register to a Mastodon Instance")
                 .arg(
-                    Arg::with_name("host")
-                        .short("H")
+                    Arg::new("host")
+                        .short('H')
                         .long("host")
                         .value_name("HOST")
                         .help("Base URL of the Mastodon instance to register to (no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .required(true)
-                        .display_order(1),
+                        .display_order(1)
+                    )
+                .arg(
+                    Arg::new("name")
+                        .short('n')
+                        .long("name")
+                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                        .num_args(1)
+                        .required(true)
+                        .display_order(2)
                 ),
         )
         .subcommand(
-            SubCommand::with_name("init")
+            Command::new("init")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to init Scootaloo DB")
                 .arg(
-                    Arg::with_name("config")
-                        .short("c")
+                    Arg::new("config")
+                        .short('c')
                         .long("config")
                         .value_name("CONFIG_FILE")
                         .help(&format!(
                             "TOML config file for scootaloo (default {})",
                             DEFAULT_CONFIG_PATH
                         ))
-                        .takes_value(true)
+                        .default_value(DEFAULT_CONFIG_PATH)
+                        .num_args(1)
                         .display_order(1),
                 ),
         )
+        .subcommand(
+            Command::new("migrate")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to migrate Scootaloo DB")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                    Arg::new("name")
+                        .short('n')
+                        .long("name")
+                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                        .num_args(1)
+                        .display_order(2)
+            )
+        )
         .get_matches();
 
     match matches.subcommand() {
-        ("register", Some(sub_m)) => {
-            register(sub_m.value_of("host").unwrap());
+        Some(("register", sub_m)) => {
+            register(
+                sub_m.get_one::<String>("host").unwrap(),
+                sub_m.get_one::<String>("name").unwrap(),
+            );
             return;
         }
-        ("init", Some(sub_m)) => {
-            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+        Some(("init", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
             init_db(&config.scootaloo.db_path).unwrap();
             return;
         }
+        Some(("migrate", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            let config_twitter_screen_name =
+                &config.mastodon.values().next().unwrap().twitter_screen_name;
+            migrate_db(
+                &config.scootaloo.db_path,
+                sub_m
+                    .get_one::<String>("name")
+                    .unwrap_or(config_twitter_screen_name),
+            )
+            .unwrap();
+            return;
+        }
         _ => (),
     }
 
-    if matches.is_present("log_level") {
-        match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
-            Ok(level) => SimpleLogger::new().with_level(level).init().unwrap(),
-            Err(e) => {
-                SimpleLogger::new()
-                    .with_level(LevelFilter::Error)
-                    .init()
-                    .unwrap();
-                error!("Unknown log level filter: {}", e);
-            }
-        };
+    if let Some(level) = matches.get_one::<String>("log_level") {
+        SimpleLogger::new()
+            .with_level(LevelFilter::from_str(level).unwrap())
+            .init()
+            .unwrap();
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+    let config = parse_toml(matches.get_one::<String>("config").unwrap());
 
     run(config);
 }

src/mastodon.rs

@@ -1,88 +1,155 @@
 use crate::config::MastodonConfig;
 
-use egg_mode::{
-    entities::{MentionEntity, UrlEntity},
-    tweet::Tweet,
-};
-use elefren::{apps::App, prelude::*, scopes::Scopes};
-use html_escape::decode_html_entities;
-use std::{borrow::Cow, collections::HashMap, io::stdin};
+use egg_mode::entities::{MentionEntity, UrlEntity};
+use megalodon::{generator, mastodon::Mastodon, megalodon::AppInputOptions};
+use regex::Regex;
+use std::{collections::HashMap, io::stdin};
 
 /// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse
-fn twitter_mentions(ums: &[MentionEntity]) -> HashMap<String, String> {
-    ums.iter()
+/// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
+/// as well
+pub fn twitter_mentions(
+    toot: &mut String,
+    ums: &[MentionEntity],
+    masto: &HashMap<String, MastodonConfig>,
+) {
+    let tm: HashMap<String, String> = ums
+        .iter()
         .map(|s| {
             (
                 format!("@{}", s.screen_name),
                 format!("@{}@twitter.com", s.screen_name),
             )
         })
-        .collect()
+        .chain(
+            masto
+                .values()
+                .filter(|s| s.mastodon_screen_name.is_some())
+                .map(|s| {
+                    (
+                        format!("@{}", s.twitter_screen_name),
+                        format!(
+                            "@{}@{}",
+                            s.mastodon_screen_name.as_ref().unwrap(),
+                            s.base.split('/').last().unwrap()
+                        ),
+                    )
+                })
+                .collect::<HashMap<String, String>>(),
+        )
+        .collect();
+
+    for (k, v) in tm {
+        *toot = toot.replace(&k, &v);
+    }
 }
 
-/// Decodes urls from UrlEntities
-fn decode_urls(urls: &[UrlEntity]) -> HashMap<String, String> {
+/// Decodes urls in toot
+pub fn decode_urls(toot: &mut String, urls: &HashMap<String, String>) {
+    for (k, v) in urls {
+        *toot = toot.replace(k, v);
+    }
+}
+
+/// Reassociates source url with destination url for rewritting
+/// this takes a Teet UrlEntity and an optional Regex
+pub fn associate_urls(urls: &[UrlEntity], re: &Option<Regex>) -> HashMap<String, String> {
     urls.iter()
         .filter(|s| s.expanded_url.is_some())
         .map(|s| {
-            (
-                s.url.to_owned(),
-                s.expanded_url.as_deref().unwrap().to_owned(),
-            )
+            (s.url.to_owned(), {
+                let mut def = s.expanded_url.as_deref().unwrap().to_owned();
+
+                if let Some(r) = re {
+                    if r.is_match(s.expanded_url.as_deref().unwrap()) {
+                        def = s.display_url.to_owned();
+                    }
+                }
+
+                def
+            })
         })
-        .collect()
+        .collect::<HashMap<String, String>>()
+}
+
+/// Replaces the commonly used services by mirrors, if asked to
+pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<String, String>) {
+    for val in urls.values_mut() {
+        for (k, v) in alts {
+            *val = val.replace(&format!("/{}/", k), &format!("/{}/", v));
+        }
+    }
+}
+
+/// Finds a Mastodon screen_name/base_url from a MastodonConfig
+pub fn find_mastodon_screen_name_by_twitter_screen_name(
+    twitter_screen_name: &str,
+    masto: &HashMap<String, MastodonConfig>,
+) -> Option<(String, String)> {
+    masto.iter().find_map(|(_, v)| {
+        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
+            Some((
+                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
+                v.base.to_owned(),
+            ))
+        } else {
+            None
+        }
+    })
+}
+
+/// Replaces the original quoted tweet by the corresponding toot
+pub fn replace_tweet_by_toot(
+    urls: &mut HashMap<String, String>,
+    twitter_screen_name: &str,
+    tweet_id: u64,
+    mastodon_screen_name: &str,
+    base_url: &str,
+    toot_id: &str,
+) {
+    for val in urls.values_mut() {
+        if val.to_lowercase().starts_with(&format!(
+            "https://twitter.com/{}/status/{}",
+            twitter_screen_name.to_lowercase(),
+            tweet_id
+        )) {
+            *val = format!("{}/@{}/{}", base_url, mastodon_screen_name, toot_id);
+        }
+    }
 }
 
 /// Gets Mastodon Data
 pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
-        base: Cow::from(masto.base.to_owned()),
-        client_id: Cow::from(masto.client_id.to_owned()),
-        client_secret: Cow::from(masto.client_secret.to_owned()),
-        redirect: Cow::from(masto.redirect.to_owned()),
-        token: Cow::from(masto.token.to_owned()),
-    };
-
-    Mastodon::from(data)
-}
-
-/// Builds toot text from tweet
-pub fn build_basic_status(tweet: &Tweet) -> String {
-    let mut toot = tweet.text.to_owned();
-
-    for decoded_url in decode_urls(&tweet.entities.urls) {
-        toot = toot.replace(&decoded_url.0, &decoded_url.1);
-    }
-
-    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions) {
-        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
-    }
-
-    decode_html_entities(&toot).to_string()
+    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
 }
 
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str) {
-    let mut builder = App::builder();
-    builder
-        .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
-        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
-        .scopes(Scopes::write_all())
-        .website(Cow::from(
-            "https://framagit.org/veretcle/scootaloo".to_string(),
-        ));
+#[tokio::main]
+pub async fn register(host: &str, screen_name: &str) {
+    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
 
-    let app = builder.build().expect("Cannot build the app");
+    let options = AppInputOptions {
+        redirect_uris: None,
+        scopes: Some(
+            [
+                "read:accounts".to_string(),
+                "write:accounts".to_string(),
+                "write:media".to_string(),
+                "write:statuses".to_string(),
+            ]
+            .to_vec(),
+        ),
+        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
+    };
 
-    let registration = Registration::new(host)
-        .register(app)
-        .expect("Cannot build registration object");
-    let url = registration
-        .authorize_url()
-        .expect("Cannot generate registration URI!");
+    let app_data = mastodon
+        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
+        .await
+        .expect("Cannot build registration object!");
+
+    let url = app_data.url.expect("Cannot generate registration URI!");
 
     println!("Click this link to authorize on Mastodon: {}", url);
     println!("Paste the returned authorization code: ");
@@ -92,16 +159,47 @@ pub fn register(host: &str) {
         .read_line(&mut input)
         .expect("Unable to read back registration code!");
 
-    let code = input.trim();
-    let mastodon = registration
-        .complete(code)
+    let token_data = mastodon
+        .fetch_access_token(
+            app_data.client_id.to_owned(),
+            app_data.client_secret.to_owned(),
+            input.trim().to_string(),
+            megalodon::default::NO_REDIRECT.to_string(),
+        )
+        .await
         .expect("Unable to create access token!");
 
-    let toml = toml::to_string(&*mastodon).unwrap();
+    let mastodon = generator(
+        megalodon::SNS::Mastodon,
+        host.to_string(),
+        Some(token_data.access_token.to_owned()),
+        None,
+    );
+
+    let current_account = mastodon
+        .verify_account_credentials()
+        .await
+        .expect("Unable to access account information!")
+        .json();
 
     println!(
-        "Please insert the following block at the end of your configuration file:\n[mastodon]\n{}",
-        toml
+        r#"Please insert the following block at the end of your configuration file:
+[mastodon.{}]
+twitter_screen_name = "{}"
+mastodon_screen_name = "{}"
+base = "{}"
+client_id = "{}"
+client_secret = "{}"
+redirect = "{}"
+token = "{}""#,
+        screen_name.to_lowercase(),
+        screen_name,
+        current_account.username,
+        host,
+        app_data.client_id,
+        app_data.client_secret,
+        app_data.redirect_uri,
+        token_data.access_token,
     );
 }
 
@@ -109,115 +207,286 @@ pub fn register(host: &str) {
 mod tests {
     use super::*;
 
-    use chrono::prelude::*;
-    use egg_mode::tweet::TweetEntities;
+    #[test]
+    fn test_replace_tweet_by_toot() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
+            ),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+        ]);
+
+        replace_tweet_by_toot(
+            &mut associated_urls,
+            "NintendojoFR",
+            1590047921633755136,
+            "nintendojofr",
+            "https://m.nintendojo.fr",
+            "109309605486908797",
+        );
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
+        let masto_config = HashMap::from([
+            (
+                "test".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tonpere".to_string(),
+                    mastodon_screen_name: Some("lalali".to_string()),
+                    twitter_page_size: None,
+                    base: "https://mstdn.net".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test2".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tamerelol".to_string(),
+                    mastodon_screen_name: None,
+                    twitter_page_size: None,
+                    base: "https://mastoot.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test3".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "NintendojoFR".to_string(),
+                    mastodon_screen_name: Some("nintendojofr".to_string()),
+                    twitter_page_size: None,
+                    base: "https://m.nintendojo.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+        ]);
+
+        // case sensitiveness, to avoid any mistake
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
+        );
+        assert_eq!(
+            Some((
+                "nintendojofr".to_string(),
+                "https://m.nintendojo.fr".to_string()
+            )),
+            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
+        );
+        // should return None if twitter_screen_name is undefined
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
+        );
+        assert_eq!(
+            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
+            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
+        );
+    }
 
     #[test]
     fn test_twitter_mentions() {
-        let mention_entity = MentionEntity {
-            id: 12345,
-            range: (1, 3),
-            name: "Ta Mere l0l".to_string(),
-            screen_name: "tamerelol".to_string(),
-        };
+        let mention_entities = vec![
+            MentionEntity {
+                id: 12345,
+                range: (1, 3),
+                name: "Ta Mere l0l".to_string(),
+                screen_name: "tamerelol".to_string(),
+            },
+            MentionEntity {
+                id: 6789,
+                range: (1, 3),
+                name: "TONPERE".to_string(),
+                screen_name: "tonpere".to_string(),
+            },
+        ];
 
-        let twitter_ums = vec![mention_entity];
+        let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
 
-        let mut expected_mentions = HashMap::new();
-        expected_mentions.insert(
-            "@tamerelol".to_string(),
-            "@tamerelol@twitter.com".to_string(),
-        );
+        let masto_config = HashMap::from([(
+            "test".to_string(),
+            (MastodonConfig {
+                twitter_screen_name: "tonpere".to_string(),
+                mastodon_screen_name: Some("lalali".to_string()),
+                twitter_page_size: None,
+                base: "https://mstdn.net".to_string(),
+                client_id: "".to_string(),
+                client_secret: "".to_string(),
+                redirect: "".to_string(),
+                token: "".to_string(),
+            }),
+        )]);
 
-        let decoded_mentions = twitter_mentions(&twitter_ums);
+        twitter_mentions(&mut toot, &mention_entities, &masto_config);
 
-        assert_eq!(expected_mentions, decoded_mentions);
+        assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
     }
 
     #[test]
     fn test_decode_urls() {
-        let url_entity1 = UrlEntity {
-            display_url: "tamerelol".to_string(),
-            expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
-            range: (1, 3),
-            url: "https://t.me/tamerelol".to_string(),
-        };
+        let urls = HashMap::from([
+            (
+                "https://t.co/thisisatest".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/nopenotinclusive".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
 
-        let url_entity2 = UrlEntity {
-            display_url: "tamerelol".to_string(),
-            expanded_url: None,
-            range: (1, 3),
-            url: "https://t.me/tamerelol".to_string(),
-        };
+        let mut toot =
+            "Rendez-vous sur https://t.co/thisisatest avec nos https://t.co/nopenotinclusive !"
+                .to_string();
 
-        let twitter_urls = vec![url_entity1, url_entity2];
+        decode_urls(&mut toot, &urls);
 
-        let mut expected_urls = HashMap::new();
-        expected_urls.insert(
-            "https://t.me/tamerelol".to_string(),
-            "https://www.nintendojo.fr/dojobar".to_string(),
+        assert_eq!(
+            &toot,
+            "Rendez-vous sur https://www.nintendojo.fr/dojobar avec nos invité.es !"
         );
-
-        let decoded_urls = decode_urls(&twitter_urls);
-
-        assert_eq!(expected_urls, decoded_urls);
     }
 
     #[test]
-    fn test_build_basic_status() {
-        let t = Tweet {
-            coordinates: None,
-            created_at: Utc::now(),
-            current_user_retweet: None,
-            display_text_range: None,
-            entities: TweetEntities {
-                hashtags: vec![],
-                symbols: vec![],
-                urls: vec![
-                        UrlEntity {
-                            display_url: "youtube.com/watch?v=w5TrSa…".to_string(),
-                            expanded_url: Some("https://www.youtube.com/watch?v=w5TrSaoYmZ8".to_string()),
-                            range: (93, 116),
-                            url: "https://t.co/zXw0FfX2Nt".to_string(),
-                        }
-                ],
-                user_mentions: vec![
-                        MentionEntity {
-                            id: 491500016,
-                            range: (80, 95),
-                            name: "Nintendo France".to_string(),
-                            screen_name: "NintendoFrance".to_string(),
-                        }
-                    ],
-                media: None,
+    fn test_associate_urls() {
+        let urls = vec![
+            UrlEntity {
+                display_url: "tamerelol".to_string(),
+                expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+                range: (1, 3),
+                url: "https://t.me/tamerelol".to_string(),
             },
-            extended_entities: None,
-            favorite_count: 0,
-            favorited: None,
-            filter_level: None,
-            id: 1491541246984306693,
-            in_reply_to_user_id: None,
-            in_reply_to_screen_name: None,
-            in_reply_to_status_id: None,
-            lang: None,
-            place: None,
-            possibly_sensitive: None,
-            quoted_status: None,
-            quoted_status_id: None,
-            retweet_count: 0,
-            retweeted: None,
-            retweeted_status: None,
-            source: None,
-            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance https://t.co/zXw0FfX2Nt".to_string(),
-            truncated: false,
-            user: None,
-            withheld_copyright: false,
-            withheld_in_countries: None,
-            withheld_scope: None,
-        };
+            UrlEntity {
+                display_url: "sadcat".to_string(),
+                expanded_url: None,
+                range: (1, 3),
+                url: "https://t.me/sadcat".to_string(),
+            },
+            UrlEntity {
+                display_url: "invité.es".to_string(),
+                expanded_url: Some("http://xn--invit-fsa.es".to_string()),
+                range: (85, 108),
+                url: "https://t.co/WAUgnpHLmo".to_string(),
+            },
+        ];
 
-        let t_out = build_basic_status(&t);
+        let expected_urls = HashMap::from([
+            (
+                "https://t.me/tamerelol".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/WAUgnpHLmo".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
 
-        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com https://www.youtube.com/watch?v=w5TrSaoYmZ8");
+        let re = Regex::new("(.+)\\.es$").ok();
+
+        let associated_urls = associate_urls(&urls, &re);
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_replace_alt_services() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://www.youtube.com/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://twitter.com/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        let alt_services = HashMap::from([
+            ("twitter.com".to_string(), "nitter.net".to_string()),
+            ("youtu.be".to_string(), "invidio.us".to_string()),
+            ("www.youtube.com".to_string(), "invidio.us".to_string()),
+            ("youtube.com".to_string(), "invidio.us".to_string()),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://invidio.us/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://nitter.net/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        replace_alt_services(&mut associated_urls, &alt_services);
+
+        assert_eq!(associated_urls, expected_urls);
     }
 }

src/state.rs

@@ -1,10 +1,13 @@
-use log::debug;
-use rusqlite::{params, Connection, OptionalExtension};
 use std::error::Error;
 
+use log::debug;
+
+use rusqlite::{params, Connection, OptionalExtension};
+
 /// Struct for each query line
 #[derive(Debug)]
 pub struct TweetToToot {
+    pub twitter_screen_name: String,
     pub tweet_id: u64,
     pub toot_id: String,
 }
@@ -13,12 +16,13 @@ pub struct TweetToToot {
 /// if a tweet_id is passed, read this particular tweet from DB
 pub fn read_state(
     conn: &Connection,
+    n: &str,
     s: Option<u64>,
 ) -> Result<Option<TweetToToot>, Box<dyn Error>> {
     debug!("Reading tweet_id {:?}", s);
     let query: String = match s {
-        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {}", i),
-        None => "SELECT * FROM tweet_to_toot ORDER BY tweet_id DESC LIMIT 1".to_string(),
+        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {} and twitter_screen_name = \"{}\"", i, n),
+        None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{}\" ORDER BY tweet_id DESC LIMIT 1", n),
     };
 
     let mut stmt = conn.prepare(&query)?;
@@ -26,8 +30,9 @@ pub fn read_state(
     let t = stmt
         .query_row([], |row| {
             Ok(TweetToToot {
-                tweet_id: row.get(0)?,
-                toot_id: row.get(1)?,
+                twitter_screen_name: row.get("twitter_screen_name")?,
+                tweet_id: row.get("tweet_id")?,
+                toot_id: row.get("toot_id")?,
             })
         })
         .optional()?;
@@ -39,8 +44,8 @@ pub fn read_state(
 pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
     debug!("Write struct {:?}", t);
     conn.execute(
-        "INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)",
-        params![t.tweet_id, t.toot_id],
+        "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id) VALUES (?1, ?2, ?3)",
+        params![t.twitter_screen_name, t.tweet_id, t.toot_id],
     )?;
 
     Ok(())
@@ -53,8 +58,9 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
 
     conn.execute(
         "CREATE TABLE IF NOT EXISTS tweet_to_toot (
-            tweet_id        INTEGER PRIMARY KEY,
-            toot_id         TEXT UNIQUE
+            twitter_screen_name TEXT NOT NULL,
+            tweet_id            INTEGER PRIMARY KEY,
+            toot_id             TEXT UNIQUE
             )",
         [],
     )?;
@@ -62,6 +68,31 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+/// Migrate DB from 0.6.x to 0.7.x
+pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
+    debug!("Migrating DB for Scootaloo");
+
+    let conn = Connection::open(d)?;
+
+    let res = conn.execute(
+        &format!(
+            "ALTER TABLE tweet_to_toot
+        ADD COLUMN twitter_screen_name TEXT NOT NULL
+        DEFAULT \"{}\"",
+            s
+        ),
+        [],
+    );
+
+    match res {
+        Err(e) => match e.to_string().as_str() {
+            "duplicate column name: twitter_screen_name" => Ok(()),
+            _ => Err(e.into()),
+        },
+        _ => Ok(()),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -93,9 +124,9 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (100, 'A');",
+            ('tamerelol', 100, 'A');",
             [],
         )
         .unwrap();
@@ -114,6 +145,7 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         let t_in = TweetToToot {
+            twitter_screen_name: "tamerelol".to_string(),
             tweet_id: 123456789,
             toot_id: "987654321".to_string(),
         };
@@ -125,14 +157,16 @@ mod tests {
         let t_out = stmt
             .query_row([], |row| {
                 Ok(TweetToToot {
-                    tweet_id: row.get(0).unwrap(),
-                    toot_id: row.get(1).unwrap(),
+                    twitter_screen_name: row.get("twitter_screen_name").unwrap(),
+                    tweet_id: row.get("tweet_id").unwrap(),
+                    toot_id: row.get("toot_id").unwrap(),
                 })
             })
             .unwrap();
 
+        assert_eq!(&t_out.twitter_screen_name, "tamerelol");
         assert_eq!(t_out.tweet_id, 123456789);
-        assert_eq!(t_out.toot_id, "987654321".to_string());
+        assert_eq!(&t_out.toot_id, "987654321");
 
         remove_file(d).unwrap();
     }
@@ -146,15 +180,15 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (101, 'A'),
-            (102, 'B');",
+            ('tamerelol', 101, 'A'),
+            ('tamerelol', 102, 'B');",
             [],
         )
         .unwrap();
 
-        let t_out = read_state(&conn, None).unwrap().unwrap();
+        let t_out = read_state(&conn, "tamerelol", None).unwrap().unwrap();
 
         remove_file(d).unwrap();
 
@@ -170,7 +204,7 @@ mod tests {
 
         let conn = Connection::open(d).unwrap();
 
-        let t_out = read_state(&conn, None).unwrap();
+        let t_out = read_state(&conn, "tamerelol", None).unwrap();
 
         remove_file(d).unwrap();
 
@@ -186,14 +220,14 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (100, 'A');",
+            ('tamerelol', 100, 'A');",
             [],
         )
         .unwrap();
 
-        let t_out = read_state(&conn, Some(101)).unwrap();
+        let t_out = read_state(&conn, "tamerelol", Some(101)).unwrap();
 
         remove_file(d).unwrap();
 
@@ -209,18 +243,62 @@ mod tests {
         let conn = Connection::open(d).unwrap();
 
         conn.execute(
-            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
             VALUES
-            (100, 'A');",
+            ('tamerelol', 100, 'A');",
             [],
         )
         .unwrap();
 
-        let t_out = read_state(&conn, Some(100)).unwrap().unwrap();
+        let t_out = read_state(&conn, "tamerelol", Some(100)).unwrap().unwrap();
 
         remove_file(d).unwrap();
 
         assert_eq!(t_out.tweet_id, 100);
         assert_eq!(t_out.toot_id, "A");
     }
+
+    #[test]
+    fn test_migrate_db_add_column() {
+        let d = "/tmp/test_migrate_db_add_column.sqlite";
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS tweet_to_toot (
+            tweet_id            INTEGER PRIMARY KEY,
+            toot_id             TEXT UNIQUE
+            )",
+            [],
+        )
+        .unwrap();
+
+        migrate_db(d, "tamerelol").unwrap();
+
+        let mut stmt = conn.prepare("PRAGMA table_info(tweet_to_toot);").unwrap();
+
+        let mut t = stmt.query([]).unwrap();
+
+        while let Some(row) = t.next().unwrap() {
+            if row.get::<usize, u8>(0).unwrap() == 2 {
+                assert_eq!(
+                    row.get::<usize, String>(1).unwrap(),
+                    "twitter_screen_name".to_string()
+                );
+            }
+        }
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_migrate_db_no_add_column() {
+        let d = "/tmp/test_migrate_db_no_add_column.sqlite";
+
+        init_db(d).unwrap();
+
+        migrate_db(d, "tamerelol").unwrap();
+
+        remove_file(d).unwrap();
+    }
 }

src/twitter.rs

@@ -27,24 +27,31 @@ pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
     }
 }
 
-/// Gets Twitter user timeline
+/// Gets Twitter user timeline, eliminate responses to others and reverse it
 pub async fn get_user_timeline(
-    config: &TwitterConfig,
-    token: Token,
+    screen_name: &str,
+    token: &Token,
     lid: Option<u64>,
+    page_size: i32,
 ) -> Result<Vec<Tweet>, Box<dyn Error>> {
     // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_, feed) = user_timeline(
-        UserID::from(config.username.to_owned()),
-        true,
-        false,
-        &token,
-    )
-    .with_page_size(200)
-    .older(lid)
-    .await?;
+    let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
+        .with_page_size(page_size)
+        .older(lid)
+        .await?;
 
-    Ok(feed.to_vec())
+    let mut feed: Vec<Tweet> = feed
+        .iter()
+        .cloned()
+        .filter(|t| match &t.in_reply_to_screen_name {
+            Some(r) => r.to_lowercase() == screen_name.to_lowercase(),
+            None => true,
+        })
+        .collect();
+
+    feed.reverse();
+
+    Ok(feed)
 }
 
 /// Retrieves a single media from a tweet and store it in a temporary file

src/util.rs

@@ -1,11 +1,76 @@
-use crate::ScootalooError;
+use crate::{twitter::get_tweet_media, ScootalooError};
+
+use egg_mode::tweet::Tweet;
+use futures::{stream, stream::StreamExt};
+use log::{error, info, warn};
+use megalodon::{mastodon::Mastodon, megalodon::Megalodon};
 use reqwest::Url;
 use std::error::Error;
 use tokio::{
-    fs::{create_dir_all, File},
+    fs::{create_dir_all, remove_file, File},
     io::copy,
 };
 
+/// Generate associative table between media ids and tweet extended entities
+pub async fn generate_media_ids(
+    tweet: &Tweet,
+    cache_path: &str,
+    mastodon: &Mastodon,
+) -> (String, Vec<String>) {
+    let mut media_url = "".to_string();
+    let mut media_ids: Vec<String> = vec![];
+
+    if let Some(m) = &tweet.extended_entities {
+        info!("{} medias in tweet", m.media.len());
+
+        let medias = m.media.clone();
+
+        let mut stream = stream::iter(medias)
+            .map(|media| {
+                // attribute media url
+                media_url = media.url.clone();
+
+                // clone everything we need
+                let cache_path = String::from(cache_path);
+                let mastodon = mastodon.clone();
+
+                tokio::task::spawn(async move {
+                    info!("Start treating {}", media.media_url_https);
+                    // get the tweet embedded media
+                    let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
+
+                    // upload media to Mastodon
+                    let mastodon_media = mastodon
+                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        .await?
+                        .json();
+                    // at this point, we can safely erase the original file
+                    // it doesn’t matter if we can’t remove, cache_media fn is idempotent
+                    remove_file(&local_tweet_media_path).await.ok();
+
+                    Ok::<String, ScootalooError>(mastodon_media.id)
+                })
+            })
+            .buffered(4); // there are max four medias per tweet and they need to be treated in
+                          // order
+
+        while let Some(result) = stream.next().await {
+            match result {
+                Ok(Ok(v)) => media_ids.push(v),
+                Ok(Err(e)) => warn!("Cannot treat media: {}", e),
+                Err(e) => error!("Something went wrong when joining the main thread: {}", e),
+            }
+        }
+    } else {
+        info!("No media in tweet");
+    }
+
+    // in case some media_ids slot remained empty due to errors, remove them
+    media_ids.retain(|x| !x.is_empty());
+
+    (media_url, media_ids)
+}
+
 /// Gets and caches Twitter Media inside the determined temp dir
 pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
     // create dir

tests/config.rs

@@ -1,4 +1,108 @@
 use scootaloo::parse_toml;
+use std::collections::HashMap;
+
+#[test]
+fn test_alt_services() {
+    let toml = parse_toml("tests/no_test_alt_services.toml");
+    assert_eq!(toml.scootaloo.alternative_services_for, None);
+
+    let toml = parse_toml("tests/test_alt_services.toml");
+    assert_eq!(
+        toml.scootaloo.alternative_services_for,
+        Some(HashMap::from([
+            ("tamere.lol".to_string(), "tonpere.mdr".to_string()),
+            ("you.pi".to_string(), "you.pla".to_string())
+        ]))
+    );
+}
+
+#[test]
+fn test_re_display() {
+    let toml = parse_toml("tests/no_show_url_as_display_url_for.toml");
+    assert_eq!(toml.scootaloo.show_url_as_display_url_for, None);
+
+    let toml = parse_toml("tests/show_url_as_display_url_for.toml");
+
+    assert_eq!(
+        toml.scootaloo.show_url_as_display_url_for,
+        Some("^(.+)\\.es$".to_string())
+    );
+}
+
+#[test]
+fn test_page_size() {
+    const DEFAULT_PAGE_SIZE: i32 = 200;
+    let toml = parse_toml("tests/page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, Some(100));
+
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42));
+
+    // this is the exact line that is used inside fn run() to determine the twitter page size
+    // passed to fn get_user_timeline()
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+    let page_size_for_1 = toml
+        .mastodon
+        .get("1")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 100);
+    assert_eq!(page_size_for_1, 42);
+
+    let toml = parse_toml("tests/no_page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, None);
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    // and same here
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 200);
+}
+
+#[test]
+fn test_parse_good_toml_rate_limit() {
+    let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml");
+
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize));
+}
+
+#[test]
+fn test_parse_good_toml_mastodon_screen_name() {
+    let parse_good_toml = parse_toml("tests/good_test_mastodon_screen_name.toml");
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("0")
+            .unwrap()
+            .mastodon_screen_name,
+        Some("tarace".to_string())
+    );
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("1")
+            .unwrap()
+            .mastodon_screen_name,
+        None
+    );
+}
 
 #[test]
 fn test_parse_good_toml() {
@@ -9,21 +113,45 @@ fn test_parse_good_toml() {
         "/var/random/scootaloo.sqlite"
     );
     assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, None);
 
-    assert_eq!(parse_good_toml.twitter.username, "tamerelol");
     assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
     assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");
     assert_eq!(parse_good_toml.twitter.access_key, "rand access key");
     assert_eq!(parse_good_toml.twitter.access_secret, "super secret");
 
-    assert_eq!(parse_good_toml.mastodon.base, "https://m.nintendojo.fr");
-    assert_eq!(parse_good_toml.mastodon.client_id, "rand client id");
-    assert_eq!(parse_good_toml.mastodon.client_secret, "secret");
     assert_eq!(
-        parse_good_toml.mastodon.redirect,
+        &parse_good_toml
+            .mastodon
+            .get("tamerelol")
+            .unwrap()
+            .twitter_screen_name,
+        "tamerelol"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().base,
+        "https://m.nintendojo.fr"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().client_id,
+        "rand client id"
+    );
+    assert_eq!(
+        &parse_good_toml
+            .mastodon
+            .get("tamerelol")
+            .unwrap()
+            .client_secret,
+        "secret"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().redirect,
         "urn:ietf:wg:oauth:2.0:oob"
     );
-    assert_eq!(parse_good_toml.mastodon.token, "super secret");
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().token,
+        "super secret"
+    );
 }
 
 #[test]

tests/good_test.toml

@@ -4,14 +4,14 @@ db_path="/var/random/scootaloo.sqlite"
 cache_path="/tmp/scootaloo"
 
 [twitter]
-username="tamerelol"
-
 consumer_key="rand consumer key"
 consumer_secret="secret"
 access_key="rand access key"
 access_secret="super secret"
 
 [mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
 base = "https://m.nintendojo.fr"
 client_id = "rand client id"
 client_secret = "secret"

tests/good_test_mastodon_screen_name.toml

@@ -0,0 +1,28 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+mastodon_screen_name="tarace"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/good_test_rate_limit.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+rate_limit=69
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_page_size.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_show_url_as_display_url_for.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_test_alt_services.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/page_size.toml

@@ -0,0 +1,29 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+page_size=100
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tonperemdr"
+twitter_page_size=42
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/show_url_as_display_url_for.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+show_url_as_display_url_for = "^(.+)\\.es$"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/test_alt_services.toml

@@ -0,0 +1,22 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr"
+"you.pi" = "you.pla"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"