Update README.md

Update file README.md

See merge request veretcle/scootaloo!57

.gitlab-ci.yml

@@ -1,15 +1,5 @@
 ---
-
+include:
-stages:
+  project: 'veretcle/ci-common'
-  - build
+  ref: 'main'
-
+  file: 'ci_rust.yml'
-rust-latest:
-  stage: build
-  artifacts:
-    paths:
-      - target/release/scootaloo
-  image: rust:latest
-  script:
-    - cargo test
-    - cargo build --release --verbose
-    - strip target/release/${CI_PROJECT_NAME}

Cargo.toml

@@ -1,21 +1,29 @@
 [package]
 name = "scootaloo"
-version = "0.4.2"
+version = "1.1.6"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+base64 = "^0.13"
+regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^2.34"
+clap = "^4"
-futures = "^0.3"
 egg-mode = "^0.16"
-tokio = { version = "1", features = ["full"]}
+rusqlite = "^0.27"
-elefren = "^0.22"
+tokio = { version = "^1", features = ["rt"]}
-htmlescape = "^0.3"
+futures = "^0.3"
+megalodon = "^0.3.6"
+html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
 simple_logger = "^2.1"
+mime = "^0.3"
 
+[profile.release]
+strip = true
+lto = true
+codegen-units = 1

README.md

@@ -1,3 +1,9 @@
+**Due to the new Twitter policy about API v1.1 and API v2, this project will no longer be updated as it no longer can work for free any way, shape or form.**
+
+First of all, I’m deeply sorry about that. It worked pretty great for what it did with a level of quality I’m very proud to have achieved.
+
+Secondly, fuck you Musk, fuck you.
+
 A Twitter to Mastodon copy bot written in Rust
 
 It:
@@ -7,37 +13,61 @@ It:
 
 If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
 
-RT are excluded, replies are included.but only the source threads are copied, not the actual replies to other Twitter users.
+RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
 
 # Usage
+## Configuring
 
 First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this:
 
-```
+```toml
 [scootaloo]
-
+db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
-last_tweet_path="/usr/local/etc/last_tweet" ## file containing the last tweet id received, must be writable
+cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
-cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneously
+## optional, this should be omitted the majority of the time
+## sometimes, twitter try to use french inclusive writting, but instead of using `·` (median point), they’re using `.`
+## this makes twitter interpret it as a URL, which is wrong
+## this parameter allows you to catch such URLs and apply the `display_url` (i.e. `tout.es`) instead of the `expanded_url` (i.e. `http://tout.es`)
+## in those particular cases
+## (!) use with caution, it might have some undesired effects
+show_url_as_display_url_for = "^https?://(.+)\\.es$"
+## optional, this allows you to replace the host for popular services such as YouTube of Twitter, or any other
+## with their more freely accessible equivalent
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr" ## quotes are necessary for both parameters
+"you.pi" = "you.pla"
+"www.you.pi" = "you.pla" ## this is an exact match, so you’ll need to lay out all the possibilities
 
 [twitter]
-username="NintendojoFR" ## User Timeline to copy
-
 ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
-consumer_key="MYCONSUMERKEY"
+page_size = 20 ## optional, default 200, max number of tweet retrieved
-consumer_secret="MYCONSUMERSECRET"
+consumer_key = "MYCONSUMERKEY"
-access_key="MYACCESSKEY"
+consumer_secret = "MYCONSUMERSECRET"
-access_secret="MYACCESSSECRET"
+access_key = "MYACCESSKEY"
+access_secret = "MYACCESSSECRET"
+
+[mastodon]
 ```
 
+Then run the command with the `init` subcommand to initiate the DB:
+```sh
+scootaloo init
+```
+
+This subcommand is completely idempotent.
+
 Then run the command with the `register` subcommand:
-```
+```sh
 scootaloo register --host https://m.nintendojo.fr
 ```
 
 This will give you the end of the TOML file. It will look like this:
 
-```
+```toml
-[mastodon]
+[mastodon.nintendojofr] ## account
+twitter_screen_name = "NintendojoFR" ## User Timeline to copy
+mastodon_screen_name = "nintendojofr" ## optional, Mastodon account name used for smart mentions
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
 client_secret = "MYCLIENTSECRET"
@@ -45,32 +75,59 @@ redirect = "urn:ietf:wg:oauth:2.0:oob"
 token = "MYTOKEN"
 ```
 
+You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
+
+You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so:
+```
+twitter_page_size = 40
+```
+
+## Running
+
 You can then run the application via `cron` for example. Here is the generic usage:
 
-```
+```sh
-USAGE:
+A Twitter to Mastodon bot
-    scootaloo [OPTIONS] [SUBCOMMAND]
 
-FLAGS:
+Usage: scootaloo [OPTIONS] [COMMAND]
-    -h, --help       Prints help information
-    -V, --version    Prints version information
 
-OPTIONS:
+Commands:
-    -c, --config <CONFIG_FILE>    TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)
+  register     Command to register to a Mastodon Instance
+  init         Command to init Scootaloo DB
+  migrate      Command to migrate Scootaloo DB
+  copyprofile  Command to copy a Twitter profile into Mastodon
+  help         Print this message or the help of the given subcommand(s)
 
-SUBCOMMANDS:
+Options:
-    help        Prints this message or the help of the given subcommand(s)
+  -c, --config <CONFIG_FILE>  TOML config file for scootaloo [default: /usr/local/etc/scootaloo.toml]
-    register    Command to register to a Mastodon Instance
+  -l, --loglevel <LOGLEVEL>   Log level [possible values: Off, Warn, Error, Info, Debug]
+  -h, --help                  Print help information
+  -V, --version               Print version information
 ```
 
 # Quirks
 
-Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the `last_tweet` file before copying an old account.
+Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the DB file before copying an old account.
 
-You can do that with a command like:
+You can insert that Tweet number, by connecting to the DB you created:
-```
+```sh
-echo -n '8189881949849' > last_tweet
+sqlite3 /var/lib/scootaloo/scootaloo.sqlite
 ```
 
-**This file should only contain the last tweet ID without any other char (no EOL or new line).**
+And inserting the data:
 
+```sql
+INSERT INTO tweet_to_toot VALUES ("<twitter_screen_name>", 1383782580412030982, "<twitter_screen_name>");
+```
+
+The last value is supposed to be the Toot ID. It cannot be null, so you better initialize it with something unique, like the Twitter Screen Name for example.
+
+# Migrating from Scootaloo ⩽ 0.6.1
+
+The DB scheme has change between version 0.6.x and 0.7.x (this is due to the multi-account nature of Scootaloo from 0.7.x onward). You need to migrate your DB. You can do so by issuing the command:
+
+```
+scootaloo migrate
+```
+
+You can optionnally specify a screen name with the `--name` option. By default, it’ll take the first screen name in the config file.

src/config.rs

@@ -0,0 +1,52 @@
+use std::{collections::HashMap, fs::read_to_string};
+
+use serde::Deserialize;
+
+/// General configuration Struct
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub twitter: TwitterConfig,
+    pub mastodon: HashMap<String, MastodonConfig>,
+    pub scootaloo: ScootalooConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TwitterConfig {
+    pub consumer_key: String,
+    pub consumer_secret: String,
+    pub access_key: String,
+    pub access_secret: String,
+    pub page_size: Option<i32>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct MastodonConfig {
+    pub twitter_screen_name: String,
+    pub mastodon_screen_name: Option<String>,
+    pub twitter_page_size: Option<i32>,
+    pub base: String,
+    pub client_id: String,
+    pub client_secret: String,
+    pub redirect: String,
+    pub token: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ScootalooConfig {
+    pub db_path: String,
+    pub cache_path: String,
+    pub rate_limit: Option<usize>,
+    pub show_url_as_display_url_for: Option<String>,
+    pub alternative_services_for: Option<HashMap<String, String>>,
+}
+
+/// Parses the TOML file into a Config Struct
+pub fn parse_toml(toml_file: &str) -> Config {
+    let toml_config = read_to_string(toml_file)
+        .unwrap_or_else(|e| panic!("Cannot open config file {toml_file}: {e}"));
+
+    let config: Config = toml::from_str(&toml_config)
+        .unwrap_or_else(|e| panic!("Cannot parse TOML file {toml_file}: {e}"));
+
+    config
+}

src/error.rs

@@ -0,0 +1,41 @@
+use std::{
+    boxed::Box,
+    convert::From,
+    error::Error,
+    fmt::{Display, Formatter, Result},
+};
+
+use megalodon::error::Error as megalodonError;
+
+#[derive(Debug)]
+pub struct ScootalooError {
+    details: String,
+}
+
+impl ScootalooError {
+    pub fn new(msg: &str) -> ScootalooError {
+        ScootalooError {
+            details: msg.to_string(),
+        }
+    }
+}
+
+impl Error for ScootalooError {}
+
+impl Display for ScootalooError {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        write!(f, "{}", self.details)
+    }
+}
+
+impl From<Box<dyn Error>> for ScootalooError {
+    fn from(error: Box<dyn Error>) -> Self {
+        ScootalooError::new(&format!("Error in a subset crate: {error}"))
+    }
+}
+
+impl From<megalodonError> for ScootalooError {
+    fn from(error: megalodonError) -> Self {
+        ScootalooError::new(&format!("Error in megalodon crate: {error}"))
+    }
+}

src/lib.rs

@@ -1,419 +1,312 @@
-// std
+mod error;
-use std::{
+use error::ScootalooError;
-    borrow::Cow,
+
-    collections::HashMap,
+mod config;
-    io::stdin,
+pub use config::parse_toml;
-    fmt,
+use config::Config;
-    fs::{read_to_string, write},
+
-    error::Error,
+mod mastodon;
+pub use mastodon::register;
+use mastodon::*;
+
+mod twitter;
+use twitter::*;
+
+mod util;
+use util::{base64_media, generate_media_ids};
+
+mod state;
+pub use state::{init_db, migrate_db};
+use state::{read_state, write_state, TweetToToot};
+
+use futures::StreamExt;
+use html_escape::decode_html_entities;
+use log::info;
+use megalodon::{
+    megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
 };
+use regex::Regex;
+use rusqlite::Connection;
+use std::sync::Arc;
+use tokio::{spawn, sync::Mutex};
 
-// toml
+const DEFAULT_RATE_LIMIT: usize = 4;
-use serde::Deserialize;
+const DEFAULT_PAGE_SIZE: i32 = 200;
-
-// egg-mode
-use egg_mode::{
-    Token,
-    KeyPair,
-    entities::{UrlEntity, MediaEntity, MentionEntity, MediaType},
-    user::UserID,
-    tweet::{
-        Tweet,
-        user_timeline,
-    },
-};
-
-// elefren
-use elefren::{
-    prelude::*,
-    apps::App,
-    status_builder::StatusBuilder,
-    scopes::Scopes,
-};
-
-// reqwest
-use reqwest::Url;
-
-// tokio
-use tokio::{
-    io::copy,
-    fs::{File, create_dir_all, remove_file},
-};
-
-// htmlescape
-use htmlescape::decode_html;
-
-// log
-use log::{info, warn, error, debug};
-
-/**********
- * Generic usage functions
-***********/
-/*
- * Those functions are related to the Twitter side of things
- */
-/// Reads last tweet id from a file
-fn read_state(s: &str) -> Option<u64> {
-    let state = read_to_string(s);
-
-    if let Ok(s) = state {
-        debug!("Last Tweet ID (from file): {}", &s);
-        return s.parse::<u64>().ok();
-    }
-
-    None
-}
-
-/// Writes last treated tweet id to a file
-fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
-    write(f, format!("{}", s))
-}
-
-/// Gets Twitter oauth2 token
-fn get_oauth2_token(config: &Config) -> Token {
-    let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
-    let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
-
-    Token::Access {
-        consumer: con_token,
-        access: access_token,
-    }
-}
-
-/// Gets Twitter user timeline
-async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
-    // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
-                .with_page_size(200)
-                .older(lid)
-                .await?;
-
-    Ok(feed.to_vec())
-}
-
-/// Decodes urls from UrlEntities
-fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
-    let mut decoded_urls = HashMap::new();
-
-    for url in urls {
-        if url.expanded_url.is_some() {
-            // unwrap is safe here as we just verified that there is something inside expanded_url
-            decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
-        }
-    }
-
-    decoded_urls
-}
-
-/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse
-fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
-    let mut decoded_mentions = HashMap::new();
-
-    for um in ums {
-        decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
-    }
-
-    decoded_mentions
-}
-
-/// Retrieves a single media from a tweet and store it in a temporary file
-async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
-    match m.media_type {
-        MediaType::Photo => {
-            return cache_media(&m.media_url_https, t).await;
-        },
-        _ => {
-            match &m.video_info {
-                Some(v) => {
-                    for variant in &v.variants {
-                        if variant.content_type == "video/mp4" {
-                            return cache_media(&variant.url, t).await;
-                        }
-                    }
-                    return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
-                },
-                None => {
-                    return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
-                },
-            }
-        },
-    };
-}
-
-/*
- * Those functions are related to the Mastodon side of things
- */
-/// Gets Mastodon Data
-fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
-        base: Cow::from(String::from(&masto.base)),
-        client_id: Cow::from(String::from(&masto.client_id)),
-        client_secret: Cow::from(String::from(&masto.client_secret)),
-        redirect: Cow::from(String::from(&masto.redirect)),
-        token: Cow::from(String::from(&masto.token)),
-    };
-
-    Mastodon::from(data)
-}
-
-/// Builds toot text from tweet
-fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
-    let mut toot = String::from(&tweet.text);
-
-    let decoded_urls = decode_urls(&tweet.entities.urls);
-
-    for decoded_url in decoded_urls {
-        toot = toot.replace(&decoded_url.0, &decoded_url.1);
-    }
-
-    let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
-
-    for decoded_mention in decoded_mentions {
-        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
-    }
-
-    if let Ok(t) = decode_html(&toot) {
-        toot = t;
-    }
-
-    Ok(toot)
-}
-
-/*
- * Generic private functions
- */
-/// Gets and caches Twitter Media inside the determined temp dir
-async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
-    // create dir
-    create_dir_all(t).await?;
-
-    // get file
-    let mut response = reqwest::get(u).await?;
-
-    // create local file
-    let url = Url::parse(u)?;
-    let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
-                    .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
-
-    let dest_filepath = format!("{}/{}", t, dest_filename);
-
-    let mut dest_file = File::create(&dest_filepath).await?;
-
-    while let Some(chunk) = response.chunk().await? {
-        copy(&mut &*chunk, &mut dest_file).await?;
-    }
-
-    Ok(dest_filepath)
-}
-
-/**********
- * local error handler
-**********/
-#[derive(Debug)]
-struct ScootalooError {
-    details: String,
-}
-
-impl ScootalooError {
-    fn new(msg: &str) -> ScootalooError {
-        ScootalooError {
-            details: String::from(msg),
-        }
-    }
-}
-
-impl fmt::Display for ScootalooError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl std::error::Error for ScootalooError {
-    fn description(&self) -> &str {
-        &self.details
-    }
-}
-
-/**********
- * Config structure
-***********/
-/// General configuration Struct
-#[derive(Debug, Deserialize)]
-pub struct Config {
-    twitter: TwitterConfig,
-    mastodon: MastodonConfig,
-    scootaloo: ScootalooConfig,
-}
-
-#[derive(Debug, Deserialize)]
-struct TwitterConfig {
-    username: String,
-    consumer_key: String,
-    consumer_secret: String,
-    access_key: String,
-    access_secret: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct MastodonConfig {
-    base: String,
-    client_id: String,
-    client_secret: String,
-    redirect: String,
-    token: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct ScootalooConfig {
-    last_tweet_path: String,
-    cache_path: String,
-}
-
-/*********
- * Main functions
-*********/
-/// Parses the TOML file into a Config Struct
-pub fn parse_toml(toml_file: &str) -> Config {
-    let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
-        panic!("Cannot open config file {}: {}", toml_file, e)
-    );
-
-    let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
-        panic!("Cannot parse TOML file {}: {}", toml_file, e)
-    );
-
-    config
-}
-
-/// Generic register function
-/// As this function is supposed to be run only once, it will panic for every error it encounters
-/// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str) {
-    let mut builder = App::builder();
-    builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
-        .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
-        .scopes(Scopes::write_all())
-        .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
-
-    let app = builder.build().expect("Cannot build the app");
-
-    let registration = Registration::new(host).register(app).expect("Cannot build registration object");
-    let url = registration.authorize_url().expect("Cannot generate registration URI!");
-
-    println!("Click this link to authorize on Mastodon: {}", url);
-    println!("Paste the returned authorization code: ");
-
-    let mut input = String::new();
-    stdin().read_line(&mut input).expect("Unable to read back registration code!");
-
-    let code = input.trim();
-    let mastodon = registration.complete(code).expect("Unable to create access token!");
-
-    let toml = toml::to_string(&*mastodon).unwrap();
-
-    println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
-}
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
-    // retrieve the last tweet ID for the username
+    // open the SQLite connection
-    let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
+    let conn = Arc::new(Mutex::new(
+        Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
+            panic!(
+                "Something went wrong when opening the DB {}: {}",
+                &config.scootaloo.db_path, e
+            )
+        }),
+    ));
 
-    // get OAuth2 token
+    let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
-    let token = get_oauth2_token(&config);
 
-    // get Mastodon instance
+    let display_url_re = config
-    let mastodon = get_mastodon_token(&config.mastodon);
+        .scootaloo
+        .show_url_as_display_url_for
+        .as_ref()
+        .map(|r|
+        // we want to panic in case the RE is not valid
+        Regex::new(r).unwrap());
 
-    // get user timeline feed (Vec<tweet>)
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
-    let mut feed = get_user_timeline(&config, token, last_tweet_id)
+        .map(|mastodon_config| {
-        .await
+            // calculate Twitter page size
-        .unwrap_or_else(|e|
+            let page_size = mastodon_config
-        panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
+                .twitter_page_size
-    );
+                .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
 
-    // empty feed -> exiting
+            // create temporary value for each task
-    if feed.is_empty() {
+            let scootaloo_cache_path = config.scootaloo.cache_path.clone();
-        info!("Nothing to retrieve since last time, exiting…");
+            let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
-        return;
+            let display_url_re = display_url_re.clone();
-    }
+            let token = get_oauth2_token(&config.twitter);
+            let task_conn = conn.clone();
+            let global_mastodon_config = global_mastodon_config.clone();
 
-    // order needs to be chronological
+            spawn(async move {
-    feed.reverse();
+                info!("Starting treating {}", &mastodon_config.twitter_screen_name);
+                // retrieve the last tweet ID for the username
+                let lconn = task_conn.lock().await;
+                let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
+                    .map(|r| r.tweet_id);
+                drop(lconn);
 
-    for tweet in &feed {
+                // get reversed, curated user timeline
-        debug!("Treating Tweet {} inside feed", tweet.id);
+                let feed = get_user_timeline(
-        // determine if the tweet is part of a thread (response to self) or a standard response
+                    &mastodon_config.twitter_screen_name,
-        if let Some(r) =  &tweet.in_reply_to_screen_name {
+                    &token,
-                if &r.to_lowercase() != &config.twitter.username.to_lowercase() {
+                    last_tweet_id,
-                    // we are responding not threading
+                    page_size,
-                    info!("Tweet is a direct response, skipping");
+                )
-                    continue;
+                .await?;
-                }
-        };
 
-        // build basic status by just yielding text and dereferencing contained urls
+                // get Mastodon instance
-        let mut status_text = match build_basic_status(tweet) {
+                let mastodon = get_mastodon_token(&mastodon_config);
-            Ok(t) => t,
-            Err(e) => {
-                error!("Could not create status from tweet {}: {}", tweet.id ,e);
-                continue;
-            },
-        };
 
-        let mut status_medias: Vec<String> = vec![];
+                for tweet in &feed {
+                    info!("Treating Tweet {} inside feed", tweet.id);
 
-       // reupload the attachments if any
+                    // basic toot text
-        if let Some(m) = &tweet.extended_entities {
+                    let mut status_text = tweet.text.clone();
-            for media in &m.media {
-                let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path).await {
-                    Ok(m) => m,
-                    Err(e) => {
-                        error!("Cannot get tweet media for {}: {}", &media.url, e);
-                        continue;
-                    },
-                };
 
-                let mastodon_media_ids = match mastodon.media(Cow::from(String::from(&local_tweet_media_path))) {
+                    // add mentions and smart mentions
-                    Ok(m) => {
+                    if !&tweet.entities.user_mentions.is_empty() {
-                        remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
+                        info!("Tweet contains mentions, add them!");
-                            warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
+                        let global_mastodon_config = global_mastodon_config.lock().await;
+                        twitter_mentions(
+                            &mut status_text,
+                            &tweet.entities.user_mentions,
+                            &global_mastodon_config,
                         );
-                        m.id
+                        drop(global_mastodon_config);
-                    },
-                    Err(e) => {
-                        error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
-                        continue;
                     }
-                };
 
-                status_medias.push(mastodon_media_ids);
+                    if !&tweet.entities.urls.is_empty() {
+                        info!("Tweet contains links, add them!");
+                        let mut associated_urls =
+                            associate_urls(&tweet.entities.urls, &display_url_re);
 
-                // last step, removing the reference to the media from with the toot’s text
+                        if let Some(q) = &tweet.quoted_status {
-                status_text = status_text.replace(&media.url, "");
+                            if let Some(u) = &q.user {
-            }
+                                info!(
+                                    "Tweet {} contains a quote, we try to find it within the DB",
+                                    tweet.id
+                                );
+                                // we know we have a quote and a user, we can lock both the
+                                // connection to DB and global_config
+                                // we will release them manually as soon as they’re useless
+                                let lconn = task_conn.lock().await;
+                                let global_mastodon_config = global_mastodon_config.lock().await;
+                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
+                                {
+                                    info!("We have found the associated toot({})", &r.toot_id);
+                                    // drop conn immediately after the request: we won’t need it
+                                    // any more and the treatment there might be time-consuming
+                                    drop(lconn);
+                                    if let Some((m, t)) =
+                                        find_mastodon_screen_name_by_twitter_screen_name(
+                                            &r.twitter_screen_name,
+                                            &global_mastodon_config,
+                                        )
+                                    {
+                                        // drop the global conf, we have all we required, no need
+                                        // to block it further
+                                        drop(global_mastodon_config);
+                                        replace_tweet_by_toot(
+                                            &mut associated_urls,
+                                            &r.twitter_screen_name,
+                                            q.id,
+                                            &m,
+                                            &t,
+                                            &r.toot_id,
+                                        );
+                                    }
+                                }
+                            }
+                        }
+
+                        if let Some(a) = &scootaloo_alt_services {
+                            replace_alt_services(&mut associated_urls, a);
+                        }
+
+                        decode_urls(&mut status_text, &associated_urls);
+                    }
+
+                    // building associative media list
+                    let (media_url, status_medias) =
+                        generate_media_ids(tweet, &scootaloo_cache_path, &mastodon).await;
+
+                    status_text = status_text.replace(&media_url, "");
+
+                    // now that the text won’t be altered anymore, we can safely remove HTML
+                    // entities
+                    status_text = decode_html_entities(&status_text).to_string();
+
+                    info!("Building corresponding Mastodon status");
+
+                    let mut post_status = PostStatusInputOptions {
+                        media_ids: None,
+                        poll: None,
+                        in_reply_to_id: None,
+                        sensitive: None,
+                        spoiler_text: None,
+                        visibility: None,
+                        scheduled_at: None,
+                        language: None,
+                        quote_id: None,
+                    };
+
+                    if !status_medias.is_empty() {
+                        post_status.media_ids = Some(status_medias);
+                    }
+
+                    // thread if necessary
+                    if tweet.in_reply_to_user_id.is_some() {
+                        let lconn = task_conn.lock().await;
+                        if let Ok(Some(r)) = read_state(
+                            &lconn,
+                            &mastodon_config.twitter_screen_name,
+                            tweet.in_reply_to_status_id,
+                        ) {
+                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
+                        }
+                        drop(lconn);
+                    }
+
+                    // language if any
+                    if let Some(l) = &tweet.lang {
+                        post_status.language = Some(l.to_string());
+                    }
+
+                    // can be activated for test purposes
+                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
+
+                    let published_status = mastodon
+                        .post_status(status_text, Some(&post_status))
+                        .await?
+                        .json();
+                    // this will return if it cannot publish the status preventing the last_tweet from
+                    // being written into db
+
+                    let ttt_towrite = TweetToToot {
+                        twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
+                        tweet_id: tweet.id,
+                        toot_id: published_status.id,
+                    };
+
+                    // write the current state (tweet ID and toot ID) to avoid copying it another time
+                    let lconn = task_conn.lock().await;
+                    write_state(&lconn, ttt_towrite)?;
+                    drop(lconn);
+                }
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    // launch and wait for every handle
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {e}"),
+            Err(e) => eprintln!("Error with thread: {e}"),
+            _ => (),
         }
-        // finished reuploading attachments, now let’s do the toot baby!
-
-        debug!("Building corresponding Mastodon status");
-        let status = StatusBuilder::new()
-            .status(&status_text)
-            .media_ids(status_medias)
-            .build()
-            .expect(&format!("Cannot build status with text {}", &status_text));
-
-        // publish status
-        // again unwrap is safe here as we are in the main thread
-        mastodon.new_status(status).unwrap();
-        // this will panic if it cannot publish the status, which is a good thing, it allows the
-        // last_tweet gathered not to be written
-
-        // write the current state (tweet ID) to avoid copying it another time
-        write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e|
-            panic!("Can’t write the last tweet retrieved: {}", e)
-        );
     }
 }
 
+/// Copies the Twitter profile into Mastodon
+#[tokio::main]
+pub async fn profile(config: Config, bot: Option<bool>) {
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            let token = get_oauth2_token(&config.twitter);
+
+            spawn(async move {
+                // get the user of the last tweet of the feed
+                let twitter_user =
+                    get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
+                        .await?
+                        .first()
+                        .ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
+                        .clone()
+                        .user
+                        .ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
+
+                let note = get_note_from_description(
+                    &twitter_user.description,
+                    &twitter_user.entities.description.urls,
+                );
+
+                let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
+
+                let display_name = Some(String::from_utf16_lossy(
+                    &twitter_user
+                        .name
+                        .encode_utf16()
+                        .take(30)
+                        .collect::<Vec<u16>>(),
+                ));
+
+                let header = match twitter_user.profile_banner_url {
+                    Some(h) => Some(base64_media(&h).await?),
+                    None => None,
+                };
+
+                let update_creds = UpdateCredentialsInputOptions {
+                    bot,
+                    display_name,
+                    note,
+                    avatar: Some(
+                        base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
+                            .await?,
+                    ),
+                    header,
+                    fields_attributes,
+                    ..Default::default()
+                };
+
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                mastodon.update_credentials(Some(&update_creds)).await?;
+
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {e}"),
+            Err(e) => eprintln!("Error with thread: {e}"),
+            _ => (),
+        }
+    }
+}

src/main.rs

@@ -1,63 +1,183 @@
-// self
+use clap::{Arg, ArgAction, Command};
+use log::LevelFilter;
 use scootaloo::*;
-
-// clap
-use clap::{App, Arg, SubCommand};
-
-// log
-use log::{LevelFilter, error};
 use simple_logger::SimpleLogger;
-
-// std
 use std::str::FromStr;
 
+const DEFAULT_CONFIG_PATH: &str = "/usr/local/etc/scootaloo.toml";
+
 fn main() {
-    let matches = App::new(env!("CARGO_PKG_NAME"))
+    let matches = Command::new(env!("CARGO_PKG_NAME"))
-                        .version(env!("CARGO_PKG_VERSION"))
+        .version(env!("CARGO_PKG_VERSION"))
-                        .about("A Twitter to Mastodon bot")
+        .about("A Twitter to Mastodon bot")
-                        .arg(Arg::with_name("config")
+        .arg(
-                             .short("c")
+            Arg::new("config")
-                             .long("config")
+                .short('c')
-                             .value_name("CONFIG_FILE")
+                .long("config")
-                             .help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)")
+                .value_name("CONFIG_FILE")
-                             .takes_value(true)
+                .help("TOML config file for scootaloo")
-                             .display_order(1))
+                .num_args(1)
-                        .arg(Arg::with_name("log_level")
+                .default_value(DEFAULT_CONFIG_PATH)
-                             .short("l")
+                .display_order(1),
-                             .long("loglevel")
+        )
-                             .value_name("LOGLEVEL")
+        .arg(
-                             .help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
+            Arg::new("log_level")
-                             .takes_value(true)
+                .short('l')
-                             .display_order(2))
+                .long("loglevel")
-                             .subcommand(SubCommand::with_name("register")
+                .value_name("LOGLEVEL")
-                                      .version(env!("CARGO_PKG_VERSION"))
+                .help("Log level")
-                                      .about("Command to register to a Mastodon Instance")
+                .num_args(1)
-                                      .arg(Arg::with_name("host")
+                .value_parser(["Off", "Warn", "Error", "Info", "Debug"])
-                                           .short("H")
+                .display_order(2),
-                                           .long("host")
+        )
-                                           .value_name("HOST")
+        .subcommand(
-                                           .help("Base URL of the Mastodon instance to register to (no default)")
+            Command::new("register")
-                                           .takes_value(true)
+                .version(env!("CARGO_PKG_VERSION"))
-                                           .required(true)
+                .about("Command to register to a Mastodon Instance")
-                                           .display_order(1)))
+                .arg(
-                        .get_matches();
+                    Arg::new("host")
-    if let Some(matches) = matches.subcommand_matches("register") {
+                        .short('H')
-        register(matches.value_of("host").unwrap());
+                        .long("host")
-        return;
+                        .value_name("HOST")
-    }
+                        .help("Base URL of the Mastodon instance to register to (no default)")
+                        .num_args(1)
+                        .required(true)
+                        .display_order(1)
+                    )
+                .arg(
+                    Arg::new("name")
+                        .short('n')
+                        .long("name")
+                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                        .num_args(1)
+                        .required(true)
+                        .display_order(2)
+                ),
+        )
+        .subcommand(
+            Command::new("init")
+                .version(env!("CARGO_PKG_VERSION"))
+                .about("Command to init Scootaloo DB")
+                .arg(
+                    Arg::new("config")
+                        .short('c')
+                        .long("config")
+                        .value_name("CONFIG_FILE")
+                        .help("TOML config file for scootaloo")
+                        .default_value(DEFAULT_CONFIG_PATH)
+                        .num_args(1)
+                        .display_order(1),
+                ),
+        )
+        .subcommand(
+            Command::new("migrate")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to migrate Scootaloo DB")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help("TOML config file for scootaloo")
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+        )
+        .subcommand(
+            Command::new("copyprofile")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to copy a Twitter profile into Mastodon")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help("TOML config file for scootaloo")
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Mastodon Config name (as seen in the config file)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+            .arg(
+                Arg::new("bot")
+                    .short('b')
+                    .long("bot")
+                    .help("Declare user as bot")
+                    .action(ArgAction::SetTrue)
+                    .display_order(3)
+            )
+        )
+        .get_matches();
 
-    if matches.is_present("log_level") {
+    match matches.subcommand() {
-        match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
+        Some(("register", sub_m)) => {
-            Ok(level) => { SimpleLogger::new().with_level(level).init().unwrap()},
+            register(
-            Err(e) => {
+                sub_m.get_one::<String>("host").unwrap(),
-                SimpleLogger::new().with_level(LevelFilter::Error).init().unwrap();
+                sub_m.get_one::<String>("name").unwrap(),
-                error!("Unknown log level filter: {}", e);
+            );
+            return;
+        }
+        Some(("init", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            init_db(&config.scootaloo.db_path).unwrap();
+            return;
+        }
+        Some(("migrate", sub_m)) => {
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            let config_twitter_screen_name =
+                &config.mastodon.values().next().unwrap().twitter_screen_name;
+            migrate_db(
+                &config.scootaloo.db_path,
+                sub_m
+                    .get_one::<String>("name")
+                    .unwrap_or(config_twitter_screen_name),
+            )
+            .unwrap();
+            return;
+        }
+        Some(("copyprofile", sub_m)) => {
+            let mut config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            // filters out the user passed in cli from the global configuration
+            if let Some(m) = sub_m.get_one::<String>("name") {
+                match config.mastodon.get(m) {
+                    Some(_) => {
+                        config.mastodon.retain(|k, _| k == m);
+                    }
+                    None => panic!("Config file does not contain conf for {}", &m),
+                }
             }
-        };
+
+            profile(config, sub_m.get_flag("bot").then_some(true));
+
+            return;
+        }
+        _ => (),
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
+    if let Some(level) = matches.get_one::<String>("log_level") {
+        SimpleLogger::new()
+            .with_level(LevelFilter::from_str(level).unwrap())
+            .init()
+            .unwrap();
+    }
+
+    let config = parse_toml(matches.get_one::<String>("config").unwrap());
 
     run(config);
 }
-

src/mastodon.rs

@@ -0,0 +1,582 @@
+use crate::config::MastodonConfig;
+
+use egg_mode::{
+    entities::{MentionEntity, UrlEntity},
+    user::UserEntityDetail,
+};
+use megalodon::{
+    generator,
+    mastodon::Mastodon,
+    megalodon::{AppInputOptions, CredentialsFieldAttribute},
+};
+use regex::Regex;
+use std::{collections::HashMap, io::stdin};
+
+/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
+/// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
+/// as well
+pub fn twitter_mentions(
+    toot: &mut String,
+    ums: &[MentionEntity],
+    masto: &HashMap<String, MastodonConfig>,
+) {
+    let tm: HashMap<String, String> = ums
+        .iter()
+        .map(|s| {
+            (
+                format!("@{}", s.screen_name),
+                format!("@{}@twitter.com", s.screen_name),
+            )
+        })
+        .chain(
+            masto
+                .values()
+                .filter(|s| s.mastodon_screen_name.is_some())
+                .map(|s| {
+                    (
+                        format!("@{}", s.twitter_screen_name),
+                        format!(
+                            "@{}@{}",
+                            s.mastodon_screen_name.as_ref().unwrap(),
+                            s.base.split('/').last().unwrap()
+                        ),
+                    )
+                })
+                .collect::<HashMap<String, String>>(),
+        )
+        .collect();
+
+    for (k, v) in tm {
+        *toot = toot.replace(&k, &v);
+    }
+}
+
+/// Decodes urls in toot
+pub fn decode_urls(toot: &mut String, urls: &HashMap<String, String>) {
+    for (k, v) in urls {
+        *toot = toot.replace(k, v);
+    }
+}
+
+/// Reassociates source url with destination url for rewritting
+/// this takes a Teet UrlEntity and an optional Regex
+pub fn associate_urls(urls: &[UrlEntity], re: &Option<Regex>) -> HashMap<String, String> {
+    urls.iter()
+        .filter(|s| s.expanded_url.is_some())
+        .map(|s| {
+            (s.url.to_owned(), {
+                let mut def = s.expanded_url.as_deref().unwrap().to_owned();
+
+                if let Some(r) = re {
+                    if r.is_match(s.expanded_url.as_deref().unwrap()) {
+                        def = s.display_url.to_owned();
+                    }
+                }
+
+                def
+            })
+        })
+        .collect::<HashMap<String, String>>()
+}
+
+/// Replaces the commonly used services by mirrors, if asked to
+pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<String, String>) {
+    for val in urls.values_mut() {
+        for (k, v) in alts {
+            *val = val.replace(&format!("/{k}/"), &format!("/{v}/"));
+        }
+    }
+}
+
+/// Finds a Mastodon screen_name/base_url from a MastodonConfig
+pub fn find_mastodon_screen_name_by_twitter_screen_name(
+    twitter_screen_name: &str,
+    masto: &HashMap<String, MastodonConfig>,
+) -> Option<(String, String)> {
+    masto.iter().find_map(|(_, v)| {
+        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
+            Some((
+                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
+                v.base.to_owned(),
+            ))
+        } else {
+            None
+        }
+    })
+}
+
+/// Replaces the original quoted tweet by the corresponding toot
+pub fn replace_tweet_by_toot(
+    urls: &mut HashMap<String, String>,
+    twitter_screen_name: &str,
+    tweet_id: u64,
+    mastodon_screen_name: &str,
+    base_url: &str,
+    toot_id: &str,
+) {
+    for val in urls.values_mut() {
+        if val.to_lowercase().starts_with(&format!(
+            "https://twitter.com/{}/status/{}",
+            twitter_screen_name.to_lowercase(),
+            tweet_id
+        )) {
+            *val = format!("{base_url}/@{mastodon_screen_name}/{toot_id}");
+        }
+    }
+}
+
+/// Gets Mastodon Data
+pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
+    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
+}
+
+/// Gets note from twitter_user description
+pub fn get_note_from_description(t: &Option<String>, urls: &[UrlEntity]) -> Option<String> {
+    t.as_ref().map(|d| {
+        let mut n = d.to_owned();
+        let a_urls = associate_urls(urls, &None);
+        decode_urls(&mut n, &a_urls);
+        n
+    })
+}
+
+/// Gets fields_attribute from UserEntityDetail
+pub fn get_attribute_from_url(
+    user_entity_detail: &Option<UserEntityDetail>,
+) -> Option<Vec<CredentialsFieldAttribute>> {
+    user_entity_detail.as_ref().and_then(|u| {
+        u.urls.first().and_then(|v| {
+            v.expanded_url.as_ref().map(|e| {
+                vec![CredentialsFieldAttribute {
+                    name: v.display_url.to_string(),
+                    value: e.to_string(),
+                }]
+            })
+        })
+    })
+}
+
+/// Generic register function
+/// As this function is supposed to be run only once, it will panic for every error it encounters
+/// Most of this function is a direct copy/paste of the official `elefren` crate
+#[tokio::main]
+pub async fn register(host: &str, screen_name: &str) {
+    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
+
+    let options = AppInputOptions {
+        redirect_uris: None,
+        scopes: Some(
+            [
+                "read:accounts".to_string(),
+                "write:accounts".to_string(),
+                "write:media".to_string(),
+                "write:statuses".to_string(),
+            ]
+            .to_vec(),
+        ),
+        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
+    };
+
+    let app_data = mastodon
+        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
+        .await
+        .expect("Cannot build registration object!");
+
+    let url = app_data.url.expect("Cannot generate registration URI!");
+
+    println!("Click this link to authorize on Mastodon: {url}");
+    println!("Paste the returned authorization code: ");
+
+    let mut input = String::new();
+    stdin()
+        .read_line(&mut input)
+        .expect("Unable to read back registration code!");
+
+    let token_data = mastodon
+        .fetch_access_token(
+            app_data.client_id.to_owned(),
+            app_data.client_secret.to_owned(),
+            input.trim().to_string(),
+            megalodon::default::NO_REDIRECT.to_string(),
+        )
+        .await
+        .expect("Unable to create access token!");
+
+    let mastodon = generator(
+        megalodon::SNS::Mastodon,
+        host.to_string(),
+        Some(token_data.access_token.to_owned()),
+        None,
+    );
+
+    let current_account = mastodon
+        .verify_account_credentials()
+        .await
+        .expect("Unable to access account information!")
+        .json();
+
+    println!(
+        r#"Please insert the following block at the end of your configuration file:
+[mastodon.{}]
+twitter_screen_name = "{}"
+mastodon_screen_name = "{}"
+base = "{}"
+client_id = "{}"
+client_secret = "{}"
+redirect = "{}"
+token = "{}""#,
+        screen_name.to_lowercase(),
+        screen_name,
+        current_account.username,
+        host,
+        app_data.client_id,
+        app_data.client_secret,
+        app_data.redirect_uri,
+        token_data.access_token,
+    );
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_attribute_from_url() {
+        let expected_credentials_field_attribute = CredentialsFieldAttribute {
+            name: "Nintendojo.fr".to_string(),
+            value: "https://www.nintendojo.fr".to_string(),
+        };
+
+        let true_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr".to_string()),
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        let false_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: None,
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        assert!(get_attribute_from_url(&None).is_none());
+        assert!(get_attribute_from_url(&Some(UserEntityDetail { urls: false_urls })).is_none());
+
+        let binding = get_attribute_from_url(&Some(UserEntityDetail { urls: true_urls })).unwrap();
+        let result_credentials_field_attribute = binding.first().unwrap();
+
+        assert_eq!(
+            result_credentials_field_attribute.name,
+            expected_credentials_field_attribute.name
+        );
+        assert_eq!(
+            result_credentials_field_attribute.value,
+            expected_credentials_field_attribute.value
+        );
+    }
+
+    #[test]
+    fn test_get_note_from_description() {
+        let urls = vec![UrlEntity {
+            display_url: "tamerelol".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+            range: (1, 3),
+            url: "https://t.me/tamerelol".to_string(),
+        }];
+
+        let some_description = Some("Youpi | https://t.me/tamerelol".to_string());
+        let none_description = None;
+
+        assert_eq!(
+            get_note_from_description(&some_description, &urls),
+            Some("Youpi | https://www.nintendojo.fr/dojobar".to_string())
+        );
+
+        assert_eq!(get_note_from_description(&none_description, &urls), None);
+    }
+
+    #[test]
+    fn test_replace_tweet_by_toot() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
+            ),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+        ]);
+
+        replace_tweet_by_toot(
+            &mut associated_urls,
+            "NintendojoFR",
+            1590047921633755136,
+            "nintendojofr",
+            "https://m.nintendojo.fr",
+            "109309605486908797",
+        );
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
+        let masto_config = HashMap::from([
+            (
+                "test".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tonpere".to_string(),
+                    mastodon_screen_name: Some("lalali".to_string()),
+                    twitter_page_size: None,
+                    base: "https://mstdn.net".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test2".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tamerelol".to_string(),
+                    mastodon_screen_name: None,
+                    twitter_page_size: None,
+                    base: "https://mastoot.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test3".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "NintendojoFR".to_string(),
+                    mastodon_screen_name: Some("nintendojofr".to_string()),
+                    twitter_page_size: None,
+                    base: "https://m.nintendojo.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+        ]);
+
+        // case sensitiveness, to avoid any mistake
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
+        );
+        assert_eq!(
+            Some((
+                "nintendojofr".to_string(),
+                "https://m.nintendojo.fr".to_string()
+            )),
+            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
+        );
+        // should return None if twitter_screen_name is undefined
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
+        );
+        assert_eq!(
+            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
+            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
+        );
+    }
+
+    #[test]
+    fn test_twitter_mentions() {
+        let mention_entities = vec![
+            MentionEntity {
+                id: 12345,
+                range: (1, 3),
+                name: "Ta Mere l0l".to_string(),
+                screen_name: "tamerelol".to_string(),
+            },
+            MentionEntity {
+                id: 6789,
+                range: (1, 3),
+                name: "TONPERE".to_string(),
+                screen_name: "tonpere".to_string(),
+            },
+        ];
+
+        let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
+
+        let masto_config = HashMap::from([(
+            "test".to_string(),
+            (MastodonConfig {
+                twitter_screen_name: "tonpere".to_string(),
+                mastodon_screen_name: Some("lalali".to_string()),
+                twitter_page_size: None,
+                base: "https://mstdn.net".to_string(),
+                client_id: "".to_string(),
+                client_secret: "".to_string(),
+                redirect: "".to_string(),
+                token: "".to_string(),
+            }),
+        )]);
+
+        twitter_mentions(&mut toot, &mention_entities, &masto_config);
+
+        assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
+    }
+
+    #[test]
+    fn test_decode_urls() {
+        let urls = HashMap::from([
+            (
+                "https://t.co/thisisatest".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/nopenotinclusive".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
+
+        let mut toot =
+            "Rendez-vous sur https://t.co/thisisatest avec nos https://t.co/nopenotinclusive !"
+                .to_string();
+
+        decode_urls(&mut toot, &urls);
+
+        assert_eq!(
+            &toot,
+            "Rendez-vous sur https://www.nintendojo.fr/dojobar avec nos invité.es !"
+        );
+    }
+
+    #[test]
+    fn test_associate_urls() {
+        let urls = vec![
+            UrlEntity {
+                display_url: "tamerelol".to_string(),
+                expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+                range: (1, 3),
+                url: "https://t.me/tamerelol".to_string(),
+            },
+            UrlEntity {
+                display_url: "sadcat".to_string(),
+                expanded_url: None,
+                range: (1, 3),
+                url: "https://t.me/sadcat".to_string(),
+            },
+            UrlEntity {
+                display_url: "invité.es".to_string(),
+                expanded_url: Some("http://xn--invit-fsa.es".to_string()),
+                range: (85, 108),
+                url: "https://t.co/WAUgnpHLmo".to_string(),
+            },
+        ];
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.me/tamerelol".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/WAUgnpHLmo".to_string(),
+                "invité.es".to_string(),
+            ),
+        ]);
+
+        let re = Regex::new("(.+)\\.es$").ok();
+
+        let associated_urls = associate_urls(&urls, &re);
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_replace_alt_services() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://www.youtube.com/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://twitter.com/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        let alt_services = HashMap::from([
+            ("twitter.com".to_string(), "nitter.net".to_string()),
+            ("youtu.be".to_string(), "invidio.us".to_string()),
+            ("www.youtube.com".to_string(), "invidio.us".to_string()),
+            ("youtube.com".to_string(), "invidio.us".to_string()),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/youplaboom".to_string(),
+                "https://invidio.us/watch?v=dQw4w9WgXcQ".to_string(),
+            ),
+            (
+                "https://t.co/thisisfine".to_string(),
+                "https://nitter.net/Nintendo/status/1594590628771688448".to_string(),
+            ),
+            (
+                "https://t.co/nopenope".to_string(),
+                "https://www.nintendojo.fr/dojobar".to_string(),
+            ),
+            (
+                "https://t.co/broken".to_string(),
+                "http://youtu.be".to_string(),
+            ),
+            (
+                "https://t.co/alsobroken".to_string(),
+                "https://youtube.com".to_string(),
+            ),
+        ]);
+
+        replace_alt_services(&mut associated_urls, &alt_services);
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+}

src/state.rs

@@ -0,0 +1,303 @@
+use std::error::Error;
+
+use log::debug;
+
+use rusqlite::{params, Connection, OptionalExtension};
+
+/// Struct for each query line
+#[derive(Debug)]
+pub struct TweetToToot {
+    pub twitter_screen_name: String,
+    pub tweet_id: u64,
+    pub toot_id: String,
+}
+
+/// if None is passed, read the last tweet from DB
+/// if a tweet_id is passed, read this particular tweet from DB
+pub fn read_state(
+    conn: &Connection,
+    n: &str,
+    s: Option<u64>,
+) -> Result<Option<TweetToToot>, Box<dyn Error>> {
+    debug!("Reading tweet_id {:?}", s);
+    let query: String = match s {
+        Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {i} and twitter_screen_name = \"{n}\""),
+        None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{n}\" ORDER BY tweet_id DESC LIMIT 1"),
+    };
+
+    let mut stmt = conn.prepare(&query)?;
+
+    let t = stmt
+        .query_row([], |row| {
+            Ok(TweetToToot {
+                twitter_screen_name: row.get("twitter_screen_name")?,
+                tweet_id: row.get("tweet_id")?,
+                toot_id: row.get("toot_id")?,
+            })
+        })
+        .optional()?;
+
+    Ok(t)
+}
+
+/// Writes last treated tweet id and toot id to the db
+pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
+    debug!("Write struct {:?}", t);
+    conn.execute(
+        "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id) VALUES (?1, ?2, ?3)",
+        params![t.twitter_screen_name, t.tweet_id, t.toot_id],
+    )?;
+
+    Ok(())
+}
+
+/// Initiates the DB from path
+pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
+    debug!("Initializing DB for Scootaloo");
+    let conn = Connection::open(d)?;
+
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS tweet_to_toot (
+            twitter_screen_name TEXT NOT NULL,
+            tweet_id            INTEGER PRIMARY KEY,
+            toot_id             TEXT UNIQUE
+            )",
+        [],
+    )?;
+
+    Ok(())
+}
+
+/// Migrate DB from 0.6.x to 0.7.x
+pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
+    debug!("Migrating DB for Scootaloo");
+
+    let conn = Connection::open(d)?;
+
+    let res = conn.execute(
+        &format!(
+            "ALTER TABLE tweet_to_toot
+        ADD COLUMN twitter_screen_name TEXT NOT NULL
+        DEFAULT \"{s}\""
+        ),
+        [],
+    );
+
+    match res {
+        Err(e) => match e.to_string().as_str() {
+            "duplicate column name: twitter_screen_name" => Ok(()),
+            _ => Err(e.into()),
+        },
+        _ => Ok(()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::{fs::remove_file, path::Path};
+
+    #[test]
+    fn test_init_db() {
+        let d = "/tmp/test_init_db.sqlite";
+
+        init_db(d).unwrap();
+
+        // check that file exist
+        assert!(Path::new(d).exists());
+
+        // open said file
+        let conn = Connection::open(d).unwrap();
+        conn.execute("SELECT * from tweet_to_toot;", []).unwrap();
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_init_init_db() {
+        // init_db fn should be idempotent so let’s test that
+        let d = "/tmp/test_init_init_db.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
+            VALUES
+            ('tamerelol', 100, 'A');",
+            [],
+        )
+        .unwrap();
+
+        init_db(d).unwrap();
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_write_state() {
+        let d = "/tmp/test_write_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        let t_in = TweetToToot {
+            twitter_screen_name: "tamerelol".to_string(),
+            tweet_id: 123456789,
+            toot_id: "987654321".to_string(),
+        };
+
+        write_state(&conn, t_in).unwrap();
+
+        let mut stmt = conn.prepare("SELECT * FROM tweet_to_toot;").unwrap();
+
+        let t_out = stmt
+            .query_row([], |row| {
+                Ok(TweetToToot {
+                    twitter_screen_name: row.get("twitter_screen_name").unwrap(),
+                    tweet_id: row.get("tweet_id").unwrap(),
+                    toot_id: row.get("toot_id").unwrap(),
+                })
+            })
+            .unwrap();
+
+        assert_eq!(&t_out.twitter_screen_name, "tamerelol");
+        assert_eq!(t_out.tweet_id, 123456789);
+        assert_eq!(&t_out.toot_id, "987654321");
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_none_to_tweet_id_read_state() {
+        let d = "/tmp/test_none_to_tweet_id_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
+            VALUES
+            ('tamerelol', 101, 'A'),
+            ('tamerelol', 102, 'B');",
+            [],
+        )
+        .unwrap();
+
+        let t_out = read_state(&conn, "tamerelol", None).unwrap().unwrap();
+
+        remove_file(d).unwrap();
+
+        assert_eq!(t_out.tweet_id, 102);
+        assert_eq!(t_out.toot_id, "B");
+    }
+
+    #[test]
+    fn test_none_to_none_read_state() {
+        let d = "/tmp/test_none_to_none_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        let t_out = read_state(&conn, "tamerelol", None).unwrap();
+
+        remove_file(d).unwrap();
+
+        assert!(t_out.is_none());
+    }
+
+    #[test]
+    fn test_tweet_id_to_none_read_state() {
+        let d = "/tmp/test_tweet_id_to_none_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
+            VALUES
+            ('tamerelol', 100, 'A');",
+            [],
+        )
+        .unwrap();
+
+        let t_out = read_state(&conn, "tamerelol", Some(101)).unwrap();
+
+        remove_file(d).unwrap();
+
+        assert!(t_out.is_none());
+    }
+
+    #[test]
+    fn test_tweet_id_to_tweet_id_read_state() {
+        let d = "/tmp/test_tweet_id_to_tweet_id_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
+            VALUES
+            ('tamerelol', 100, 'A');",
+            [],
+        )
+        .unwrap();
+
+        let t_out = read_state(&conn, "tamerelol", Some(100)).unwrap().unwrap();
+
+        remove_file(d).unwrap();
+
+        assert_eq!(t_out.tweet_id, 100);
+        assert_eq!(t_out.toot_id, "A");
+    }
+
+    #[test]
+    fn test_migrate_db_add_column() {
+        let d = "/tmp/test_migrate_db_add_column.sqlite";
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS tweet_to_toot (
+            tweet_id            INTEGER PRIMARY KEY,
+            toot_id             TEXT UNIQUE
+            )",
+            [],
+        )
+        .unwrap();
+
+        migrate_db(d, "tamerelol").unwrap();
+
+        let mut stmt = conn.prepare("PRAGMA table_info(tweet_to_toot);").unwrap();
+
+        let mut t = stmt.query([]).unwrap();
+
+        while let Some(row) = t.next().unwrap() {
+            if row.get::<usize, u8>(0).unwrap() == 2 {
+                assert_eq!(
+                    row.get::<usize, String>(1).unwrap(),
+                    "twitter_screen_name".to_string()
+                );
+            }
+        }
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_migrate_db_no_add_column() {
+        let d = "/tmp/test_migrate_db_no_add_column.sqlite";
+
+        init_db(d).unwrap();
+
+        migrate_db(d, "tamerelol").unwrap();
+
+        remove_file(d).unwrap();
+    }
+}

src/twitter.rs

@@ -0,0 +1,193 @@
+use crate::config::TwitterConfig;
+use crate::util::cache_media;
+use crate::ScootalooError;
+
+use egg_mode::{
+    entities::{MediaEntity, MediaType},
+    tweet::{user_timeline, Tweet},
+    user::UserID,
+    KeyPair, Token,
+};
+use std::error::Error;
+
+/// Gets Twitter oauth2 token
+pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
+    let con_token = KeyPair::new(
+        config.consumer_key.to_owned(),
+        config.consumer_secret.to_owned(),
+    );
+    let access_token = KeyPair::new(
+        config.access_key.to_owned(),
+        config.access_secret.to_owned(),
+    );
+
+    Token::Access {
+        consumer: con_token,
+        access: access_token,
+    }
+}
+
+/// Gets Twitter user timeline, eliminate responses to others and reverse it
+pub async fn get_user_timeline(
+    screen_name: &str,
+    token: &Token,
+    lid: Option<u64>,
+    page_size: i32,
+) -> Result<Vec<Tweet>, Box<dyn Error>> {
+    // fix the page size to 200 as it is the maximum Twitter authorizes
+    let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
+        .with_page_size(page_size)
+        .older(lid)
+        .await?;
+
+    let mut feed: Vec<Tweet> = feed
+        .iter()
+        .cloned()
+        .filter(|t| match &t.in_reply_to_screen_name {
+            Some(r) => r.to_lowercase() == screen_name.to_lowercase(),
+            None => true,
+        })
+        .collect();
+
+    feed.reverse();
+
+    Ok(feed)
+}
+
+/// Retrieves a single media from a tweet and store it in a temporary file
+pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
+    match m.media_type {
+        MediaType::Photo => cache_media(&m.media_url_https, t).await,
+        _ => match &m.video_info {
+            Some(v) => match &v.variants.iter().find(|&x| x.content_type == "video/mp4") {
+                Some(u) => cache_media(&u.url, t).await,
+                None => Err(ScootalooError::new(&format!(
+                    "Media Type for {} is video but no mp4 file URL is available",
+                    &m.url
+                ))
+                .into()),
+            },
+            None => Err(ScootalooError::new(&format!(
+                "Media Type for {} is video but does not contain any video_info",
+                &m.url
+            ))
+            .into()),
+        },
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use egg_mode::entities::{
+        MediaSize, MediaSizes,
+        MediaType::{Gif, Photo},
+        ResizeMode::Crop,
+        ResizeMode::Fit,
+        VideoInfo, VideoVariant,
+    };
+    use std::fs::remove_dir_all;
+
+    const TMP_DIR: &'static str = "/tmp/scootaloo_get_tweet_media_test";
+
+    #[tokio::test]
+    async fn test_get_tweet_media() {
+        let m_photo = MediaEntity {
+            display_url: "pic.twitter.com/sHrwmP69Yv".to_string(),
+            expanded_url: "https://twitter.com/NintendojoFR/status/1555473821121056771/photo/1"
+                .to_string(),
+            id: 1555473771280080896,
+            range: (91, 114),
+            media_url: "http://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg".to_string(),
+            media_url_https: "https://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg"
+                .to_string(),
+            sizes: MediaSizes {
+                thumb: MediaSize {
+                    w: 150,
+                    h: 150,
+                    resize: Crop
+                },
+                small: MediaSize {
+                    w: 680,
+                    h: 510,
+                    resize: Fit
+                },
+                medium: MediaSize {
+                    w: 1200,
+                    h: 900,
+                    resize: Fit
+                },
+                large: MediaSize {
+                    w: 1280,
+                    h: 960,
+                    resize: Fit
+                }
+            },
+            source_status_id: None,
+            media_type: Photo,
+            url: "https://t.co/sHrwmP69Yv".to_string(),
+            video_info: None,
+            ext_alt_text: Some("Le menu «\u{a0}Classes » du jeu vidéo Xenoblade Chronicles 3 (Switch). L’affinité du personnage pour la classe est notée par quatre lettres : C, A, C, A (caca)."
+                .to_string())
+        };
+        let m_video = MediaEntity {
+            display_url: "pic.twitter.com/xDln0RrkjU".to_string(),
+            expanded_url: "https://twitter.com/NintendojoFR/status/1551822196833673218/photo/1"
+                .to_string(),
+            id: 1551822189711790081,
+            range: (275, 298),
+            media_url: "http://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg".to_string(),
+            media_url_https: "https://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg"
+                .to_string(),
+            sizes: MediaSizes {
+                thumb: MediaSize {
+                    w: 150,
+                    h: 150,
+                    resize: Crop,
+                },
+                small: MediaSize {
+                    w: 320,
+                    h: 240,
+                    resize: Fit,
+                },
+                medium: MediaSize {
+                    w: 320,
+                    h: 240,
+                    resize: Fit,
+                },
+                large: MediaSize {
+                    w: 320,
+                    h: 240,
+                    resize: Fit,
+                },
+            },
+            source_status_id: None,
+            media_type: Gif,
+            url: "https://t.co/xDln0RrkjU".to_string(),
+            video_info: Some(VideoInfo {
+                aspect_ratio: (4, 3),
+                duration_millis: None,
+                variants: vec![VideoVariant {
+                    bitrate: Some(0),
+                    content_type: "video/mp4".parse::<mime::Mime>().unwrap(),
+                    url: "https://video.twimg.com/tweet_video/FYkuD0RXEAE-iDx.mp4".to_string(),
+                }],
+            }),
+            ext_alt_text: Some("Scared Nintendo GIF".to_string()),
+        };
+
+        let tweet_media_photo = get_tweet_media(&m_photo, TMP_DIR).await.unwrap();
+        let tweet_media_video = get_tweet_media(&m_video, TMP_DIR).await.unwrap();
+
+        assert_eq!(
+            tweet_media_photo,
+            format!("{}/FZYnJ1qWIAAReHt.jpg", TMP_DIR)
+        );
+        assert_eq!(
+            tweet_media_video,
+            format!("{}/FYkuD0RXEAE-iDx.mp4", TMP_DIR)
+        );
+
+        remove_dir_all(TMP_DIR).unwrap();
+    }
+}

src/util.rs

@@ -0,0 +1,194 @@
+use crate::{twitter::get_tweet_media, ScootalooError};
+
+use base64::encode;
+use egg_mode::tweet::Tweet;
+use futures::{stream, stream::StreamExt};
+use log::{error, info, warn};
+use megalodon::{
+    entities::UploadMedia::{AsyncAttachment, Attachment},
+    error,
+    mastodon::Mastodon,
+    megalodon::Megalodon,
+};
+use reqwest::Url;
+use std::error::Error;
+use tokio::{
+    fs::{create_dir_all, remove_file, File},
+    io::copy,
+    time::{sleep, Duration},
+};
+
+/// Generate associative table between media ids and tweet extended entities
+pub async fn generate_media_ids(
+    tweet: &Tweet,
+    cache_path: &str,
+    mastodon: &Mastodon,
+) -> (String, Vec<String>) {
+    let mut media_url = "".to_string();
+    let mut media_ids: Vec<String> = vec![];
+
+    if let Some(m) = &tweet.extended_entities {
+        info!("{} medias in tweet", m.media.len());
+
+        let medias = m.media.clone();
+
+        let mut stream = stream::iter(medias)
+            .map(|media| {
+                // attribute media url
+                media_url = media.url.clone();
+
+                // clone everything we need
+                let cache_path = String::from(cache_path);
+                let mastodon = mastodon.clone();
+
+                tokio::task::spawn(async move {
+                    info!("Start treating {}", media.media_url_https);
+                    // get the tweet embedded media
+                    let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
+
+                    // upload media to Mastodon
+                    let mastodon_media = mastodon
+                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        .await?
+                        .json();
+                    // at this point, we can safely erase the original file
+                    // it doesn’t matter if we can’t remove, cache_media fn is idempotent
+                    remove_file(&local_tweet_media_path).await.ok();
+
+                    let id = match mastodon_media {
+                        Attachment(m) => m.id,
+                        AsyncAttachment(m) => wait_until_uploaded(&mastodon, &m.id).await?,
+                    };
+
+                    Ok::<String, ScootalooError>(id)
+                })
+            })
+            .buffered(4); // there are max four medias per tweet and they need to be treated in
+                          // order
+
+        while let Some(result) = stream.next().await {
+            match result {
+                Ok(Ok(v)) => media_ids.push(v),
+                Ok(Err(e)) => warn!("Cannot treat media: {}", e),
+                Err(e) => error!("Something went wrong when joining the main thread: {}", e),
+            }
+        }
+    } else {
+        info!("No media in tweet");
+    }
+
+    // in case some media_ids slot remained empty due to errors, remove them
+    media_ids.retain(|x| !x.is_empty());
+
+    (media_url, media_ids)
+}
+
+/// Wait on uploaded medias when necessary
+async fn wait_until_uploaded(client: &Mastodon, id: &str) -> Result<String, error::Error> {
+    loop {
+        sleep(Duration::from_secs(1)).await;
+        let res = client.get_media(id.to_string()).await;
+        return match res {
+            Ok(res) => Ok(res.json.id),
+            Err(err) => match err {
+                error::Error::OwnError(ref own_err) => match own_err.kind {
+                    error::Kind::HTTPPartialContentError => continue,
+                    _ => Err(err),
+                },
+                _ => Err(err),
+            },
+        };
+    }
+}
+
+/// Transforms the media into a base64 equivalent
+pub async fn base64_media(u: &str) -> Result<String, Box<dyn Error>> {
+    let mut response = reqwest::get(u).await?;
+
+    let mut buffer = Vec::new();
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut buffer).await?;
+    }
+
+    let content_type = response
+        .headers()
+        .get("content-type")
+        .ok_or_else(|| ScootalooError::new(&format!("Cannot get media content type for {u}")))?
+        .to_str()?;
+
+    let encoded_f = encode(buffer);
+
+    Ok(format!("data:{content_type};base64,{encoded_f}"))
+}
+
+/// Gets and caches Twitter Media inside the determined temp dir
+pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
+    // create dir
+    create_dir_all(t).await?;
+
+    // get file
+    let mut response = reqwest::get(u).await?;
+
+    // create local file
+    let url = Url::parse(u)?;
+    let dest_filename = url
+        .path_segments()
+        .ok_or_else(|| {
+            ScootalooError::new(&format!(
+                "Cannot determine the destination filename for {u}"
+            ))
+        })?
+        .last()
+        .ok_or_else(|| {
+            ScootalooError::new(&format!(
+                "Cannot determine the destination filename for {u}"
+            ))
+        })?;
+
+    let dest_filepath = format!("{t}/{dest_filename}");
+
+    let mut dest_file = File::create(&dest_filepath).await?;
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut dest_file).await?;
+    }
+
+    Ok(dest_filepath)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::{fs::remove_dir_all, path::Path};
+
+    const TMP_DIR: &'static str = "/tmp/scootaloo_test";
+
+    #[tokio::test]
+    async fn test_cache_media() {
+        let dest = cache_media(
+            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
+            TMP_DIR,
+        )
+        .await
+        .unwrap();
+
+        assert!(Path::new(&dest).exists());
+
+        remove_dir_all(TMP_DIR).unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_base64_media() {
+        let img = base64_media(
+            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
+        )
+        .await
+        .unwrap();
+
+        assert!(img.starts_with("data:image/png;base64,"));
+
+        assert!(img.ends_with("="));
+    }
+}

tests/bad_test.toml

@@ -0,0 +1 @@
+blah

tests/config.rs

@@ -0,0 +1,171 @@
+use scootaloo::parse_toml;
+use std::collections::HashMap;
+
+#[test]
+fn test_alt_services() {
+    let toml = parse_toml("tests/no_test_alt_services.toml");
+    assert_eq!(toml.scootaloo.alternative_services_for, None);
+
+    let toml = parse_toml("tests/test_alt_services.toml");
+    assert_eq!(
+        toml.scootaloo.alternative_services_for,
+        Some(HashMap::from([
+            ("tamere.lol".to_string(), "tonpere.mdr".to_string()),
+            ("you.pi".to_string(), "you.pla".to_string())
+        ]))
+    );
+}
+
+#[test]
+fn test_re_display() {
+    let toml = parse_toml("tests/no_show_url_as_display_url_for.toml");
+    assert_eq!(toml.scootaloo.show_url_as_display_url_for, None);
+
+    let toml = parse_toml("tests/show_url_as_display_url_for.toml");
+
+    assert_eq!(
+        toml.scootaloo.show_url_as_display_url_for,
+        Some("^(.+)\\.es$".to_string())
+    );
+}
+
+#[test]
+fn test_page_size() {
+    const DEFAULT_PAGE_SIZE: i32 = 200;
+    let toml = parse_toml("tests/page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, Some(100));
+
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42));
+
+    // this is the exact line that is used inside fn run() to determine the twitter page size
+    // passed to fn get_user_timeline()
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+    let page_size_for_1 = toml
+        .mastodon
+        .get("1")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 100);
+    assert_eq!(page_size_for_1, 42);
+
+    let toml = parse_toml("tests/no_page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, None);
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    // and same here
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 200);
+}
+
+#[test]
+fn test_parse_good_toml_rate_limit() {
+    let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml");
+
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize));
+}
+
+#[test]
+fn test_parse_good_toml_mastodon_screen_name() {
+    let parse_good_toml = parse_toml("tests/good_test_mastodon_screen_name.toml");
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("0")
+            .unwrap()
+            .mastodon_screen_name,
+        Some("tarace".to_string())
+    );
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("1")
+            .unwrap()
+            .mastodon_screen_name,
+        None
+    );
+}
+
+#[test]
+fn test_parse_good_toml() {
+    let parse_good_toml = parse_toml("tests/good_test.toml");
+
+    assert_eq!(
+        parse_good_toml.scootaloo.db_path,
+        "/var/random/scootaloo.sqlite"
+    );
+    assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, None);
+
+    assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
+    assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");
+    assert_eq!(parse_good_toml.twitter.access_key, "rand access key");
+    assert_eq!(parse_good_toml.twitter.access_secret, "super secret");
+
+    assert_eq!(
+        &parse_good_toml
+            .mastodon
+            .get("tamerelol")
+            .unwrap()
+            .twitter_screen_name,
+        "tamerelol"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().base,
+        "https://m.nintendojo.fr"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().client_id,
+        "rand client id"
+    );
+    assert_eq!(
+        &parse_good_toml
+            .mastodon
+            .get("tamerelol")
+            .unwrap()
+            .client_secret,
+        "secret"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().redirect,
+        "urn:ietf:wg:oauth:2.0:oob"
+    );
+    assert_eq!(
+        &parse_good_toml.mastodon.get("tamerelol").unwrap().token,
+        "super secret"
+    );
+}
+
+#[test]
+#[should_panic(
+    expected = "Cannot open config file tests/no_file.toml: No such file or directory (os error 2)"
+)]
+fn test_parse_no_toml() {
+    let _parse_no_toml = parse_toml("tests/no_file.toml");
+}
+
+#[test]
+#[should_panic(
+    expected = "Cannot parse TOML file tests/bad_test.toml: expected an equals, found a newline at line 1 column 5"
+)]
+fn test_parse_bad_toml() {
+    let _parse_bad_toml = parse_toml("tests/bad_test.toml");
+}

tests/good_test.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/good_test_mastodon_screen_name.toml

@@ -0,0 +1,28 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+mastodon_screen_name="tarace"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/good_test_rate_limit.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+rate_limit=69
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_page_size.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_show_url_as_display_url_for.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_test_alt_services.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/page_size.toml

@@ -0,0 +1,29 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+page_size=100
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tonperemdr"
+twitter_page_size=42
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/show_url_as_display_url_for.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+show_url_as_display_url_for = "^(.+)\\.es$"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/test_alt_services.toml

@@ -0,0 +1,22 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+[scootaloo.alternative_services_for]
+"tamere.lol" = "tonpere.mdr"
+"you.pi" = "you.pla"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"