Merge branch 'fix_french_inclusive_writing' into 'master'

feat: allow user to remove some links, replace some links by others

See merge request veretcle/scootaloo!38

Cargo.toml

@@ -1,22 +1,22 @@
 [package]
 name = "scootaloo"
-version = "1.0.0"
+version = "0.11.0"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+chrono = "^0.4"
 regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
 clap = "^4"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-isolang = "^2"
+tokio = { version = "^1", features = ["full"]}
-tokio = { version = "^1", features = ["rt"]}
 futures = "^0.3"
-megalodon = "^0.2"
+elefren = "^0.22"
 html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
@@ -25,4 +25,3 @@ mime = "^0.3"
 
 [profile.release]
 strip = true
-lto = true

src/config.rs

@@ -19,7 +19,7 @@ pub struct TwitterConfig {
     pub page_size: Option<i32>,
 }
 
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize)]
 pub struct MastodonConfig {
     pub twitter_screen_name: String,
     pub mastodon_screen_name: Option<String>,

src/error.rs

@@ -5,7 +5,7 @@ use std::{
     fmt::{Display, Formatter, Result},
 };
 
-use megalodon::error::Error as megalodonError;
+use elefren::Error as elefrenError;
 
 #[derive(Debug)]
 pub struct ScootalooError {
@@ -34,8 +34,8 @@ impl From<Box<dyn Error>> for ScootalooError {
     }
 }
 
-impl From<megalodonError> for ScootalooError {
+impl From<elefrenError> for ScootalooError {
-    fn from(error: megalodonError) -> Self {
+    fn from(error: elefrenError) -> Self {
-        ScootalooError::new(&format!("Error in megalodon crate: {}", error))
+        ScootalooError::new(&format!("Error in elefren crate: {}", error))
     }
 }

src/lib.rs

@@ -7,7 +7,7 @@ use config::Config;
 
 mod mastodon;
 pub use mastodon::register;
-use mastodon::*;
+use mastodon::{build_basic_status, get_mastodon_token};
 
 mod twitter;
 use twitter::*;
@@ -19,14 +19,12 @@ mod state;
 pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
+use elefren::{prelude::*, status_builder::StatusBuilder, Language};
 use futures::StreamExt;
-use html_escape::decode_html_entities;
-use isolang::Language;
 use log::info;
-use megalodon::{megalodon::PostStatusInputOptions, Megalodon};
 use regex::Regex;
 use rusqlite::Connection;
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 use tokio::{spawn, sync::Mutex};
 
 const DEFAULT_RATE_LIMIT: usize = 4;
@@ -45,7 +43,21 @@ pub async fn run(config: Config) {
         }),
     ));
 
-    let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
+    let scootaloo_mentions: HashMap<String, String> = config
+        .mastodon
+        .values()
+        .filter(|s| s.mastodon_screen_name.is_some())
+        .map(|s| {
+            (
+                format!("@{}", s.twitter_screen_name),
+                format!(
+                    "@{}@{}",
+                    s.mastodon_screen_name.as_ref().unwrap(),
+                    s.base.split('/').last().unwrap()
+                ),
+            )
+        })
+        .collect();
 
     let display_url_re = config
         .scootaloo
@@ -64,11 +76,11 @@ pub async fn run(config: Config) {
 
             // create temporary value for each task
             let scootaloo_cache_path = config.scootaloo.cache_path.clone();
+            let scootaloo_mentions = scootaloo_mentions.clone();
             let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
             let display_url_re = display_url_re.clone();
             let token = get_oauth2_token(&config.twitter);
             let task_conn = conn.clone();
-            let global_mastodon_config = global_mastodon_config.clone();
 
             spawn(async move {
                 info!("Starting treating {}", &mastodon_config.twitter_screen_name);
@@ -93,71 +105,26 @@ pub async fn run(config: Config) {
                 for tweet in &feed {
                     info!("Treating Tweet {} inside feed", tweet.id);
 
-                    // basic toot text
-                    let mut status_text = tweet.text.clone();
-
-                    // add mentions and smart mentions
-                    if !&tweet.entities.user_mentions.is_empty() {
-                        info!("Tweet contains mentions, add them!");
-                        let global_mastodon_config = global_mastodon_config.lock().await;
-                        twitter_mentions(
-                            &mut status_text,
-                            &tweet.entities.user_mentions,
-                            &global_mastodon_config,
-                        );
-                        drop(global_mastodon_config);
-                    }
-
-                    if !&tweet.entities.urls.is_empty() {
-                        info!("Tweet contains links, add them!");
-                        let mut associated_urls =
-                            associate_urls(&tweet.entities.urls, &display_url_re);
-
-                        if let Some(q) = &tweet.quoted_status {
-                            if let Some(u) = &q.user {
-                                info!(
-                                    "Tweet {} contains a quote, we try to find it within the DB",
-                                    tweet.id
-                                );
-                                // we know we have a quote and a user, we can lock both the
-                                // connection to DB and global_config
-                                // we will release them manually as soon as they’re useless
                     let lconn = task_conn.lock().await;
-                                let global_mastodon_config = global_mastodon_config.lock().await;
+                    // initiate the toot_reply_id var and retrieve the corresponding toot_id
-                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
+                    let toot_reply_id: Option<String> = tweet.in_reply_to_user_id.and_then(|_| {
-                                {
+                        read_state(
-                                    info!("We have found the associated toot({})", &r.toot_id);
+                            &lconn,
-                                    // drop conn immediately after the request: we won’t need it
+                            &mastodon_config.twitter_screen_name,
-                                    // any more and the treatment there might be time-consuming
+                            tweet.in_reply_to_status_id,
-                                    drop(lconn);
-                                    if let Some((m, t)) =
-                                        find_mastodon_screen_name_by_twitter_screen_name(
-                                            &r.twitter_screen_name,
-                                            &global_mastodon_config,
                         )
-                                    {
+                        .unwrap_or(None)
-                                        // drop the global conf, we have all we required, no need
+                        .map(|s| s.toot_id)
-                                        // to block it further
+                    });
-                                        drop(global_mastodon_config);
+                    drop(lconn);
-                                        replace_tweet_by_toot(
+
-                                            &mut associated_urls,
+                    // build basic status by just yielding text and dereferencing contained urls
-                                            &r.twitter_screen_name,
+                    let mut status_text = build_basic_status(
-                                            q.id,
+                        tweet,
-                                            &m,
+                        &scootaloo_mentions,
-                                            &t,
+                        &display_url_re,
-                                            &r.toot_id,
+                        &scootaloo_alt_services,
                     );
-                                    }
-                                }
-                            }
-                        }
-
-                        if let Some(a) = &scootaloo_alt_services {
-                            replace_alt_services(&mut associated_urls, a);
-                        }
-
-                        decode_urls(&mut status_text, &associated_urls);
-                    }
 
                     // building associative media list
                     let (media_url, status_medias) =
@@ -165,55 +132,30 @@ pub async fn run(config: Config) {
 
                     status_text = status_text.replace(&media_url, "");
 
-                    // now that the text won’t be altered anymore, we can safely remove HTML
-                    // entities
-                    status_text = decode_html_entities(&status_text).to_string();
-
                     info!("Building corresponding Mastodon status");
 
-                    let mut post_status = PostStatusInputOptions {
+                    let mut status_builder = StatusBuilder::new();
-                        media_ids: None,
-                        poll: None,
-                        in_reply_to_id: None,
-                        sensitive: None,
-                        spoiler_text: None,
-                        visibility: None,
-                        scheduled_at: None,
-                        language: None,
-                        quote_id: None,
-                    };
 
-                    if !status_medias.is_empty() {
+                    status_builder.status(&status_text).media_ids(status_medias);
-                        post_status.media_ids = Some(status_medias);
-                    }
 
-                    // thread if necessary
+                    // theard if necessary
-                    if tweet.in_reply_to_user_id.is_some() {
+                    if let Some(i) = toot_reply_id {
-                        let lconn = task_conn.lock().await;
+                        status_builder.in_reply_to(&i);
-                        if let Ok(Some(r)) = read_state(
-                            &lconn,
-                            &mastodon_config.twitter_screen_name,
-                            tweet.in_reply_to_status_id,
-                        ) {
-                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
-                        }
-                        drop(lconn);
                     }
 
                     // language if any
                     if let Some(l) = &tweet.lang {
                         if let Some(r) = Language::from_639_1(l) {
-                            post_status.language = Some(r.to_string());
+                            status_builder.language(r);
                         }
                     }
 
                     // can be activated for test purposes
-                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
+                    // status_builder.visibility(elefren::status_builder::Visibility::Private);
 
-                    let published_status = mastodon
+                    let status = status_builder.build()?;
-                        .post_status(status_text, Some(&post_status))
+
-                        .await?
+                    let published_status = mastodon.new_status(status)?;
-                        .json();
                     // this will return if it cannot publish the status preventing the last_tweet from
                     // being written into db
 

src/mastodon.rs

@@ -1,155 +1,123 @@
 use crate::config::MastodonConfig;
 
-use egg_mode::entities::{MentionEntity, UrlEntity};
+use egg_mode::{
-use megalodon::{generator, mastodon::Mastodon, megalodon::AppInputOptions};
+    entities::{MentionEntity, UrlEntity},
+    tweet::Tweet,
+};
+use elefren::{apps::App, prelude::*, scopes::Read, scopes::Scopes, scopes::Write};
+use html_escape::decode_html_entities;
 use regex::Regex;
-use std::{collections::HashMap, io::stdin};
+use std::{borrow::Cow, collections::HashMap, io::stdin};
 
 /// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
+/// Fediverse
-/// as well
+fn twitter_mentions(ums: &[MentionEntity]) -> HashMap<String, String> {
-pub fn twitter_mentions(
+    ums.iter()
-    toot: &mut String,
-    ums: &[MentionEntity],
-    masto: &HashMap<String, MastodonConfig>,
-) {
-    let tm: HashMap<String, String> = ums
-        .iter()
         .map(|s| {
             (
                 format!("@{}", s.screen_name),
                 format!("@{}@twitter.com", s.screen_name),
             )
         })
-        .chain(
+        .collect()
-            masto
-                .values()
-                .filter(|s| s.mastodon_screen_name.is_some())
-                .map(|s| {
-                    (
-                        format!("@{}", s.twitter_screen_name),
-                        format!(
-                            "@{}@{}",
-                            s.mastodon_screen_name.as_ref().unwrap(),
-                            s.base.split('/').last().unwrap()
-                        ),
-                    )
-                })
-                .collect::<HashMap<String, String>>(),
-        )
-        .collect();
-
-    for (k, v) in tm {
-        *toot = toot.replace(&k, &v);
-    }
 }
 
-/// Decodes urls in toot
+/// Decodes urls from UrlEntities
-pub fn decode_urls(toot: &mut String, urls: &HashMap<String, String>) {
+fn decode_urls(
-    for (k, v) in urls {
+    urls: &[UrlEntity],
-        *toot = toot.replace(k, v);
+    re: &Option<Regex>,
-    }
+    alt_urls: &Option<HashMap<String, String>>,
-}
+) -> HashMap<String, String> {
-
-/// Reassociates source url with destination url for rewritting
-/// this takes a Teet UrlEntity and an optional Regex
-pub fn associate_urls(urls: &[UrlEntity], re: &Option<Regex>) -> HashMap<String, String> {
     urls.iter()
         .filter(|s| s.expanded_url.is_some())
         .map(|s| {
             (s.url.to_owned(), {
                 let mut def = s.expanded_url.as_deref().unwrap().to_owned();
 
-                if let Some(r) = re {
+                if let Some(a) = &alt_urls {
+                    for (url_source, url_destination) in a {
+                        def = def.replace(
+                            &format!("https://{}", url_source),
+                            &format!("https://{}", url_destination),
+                        );
+                    }
+                }
+
+                if let Some(r) = &re {
                     if r.is_match(s.expanded_url.as_deref().unwrap()) {
-                        def = s.display_url.to_owned();
+                        def = s.display_url.clone();
                     }
                 }
 
                 def
             })
         })
-        .collect::<HashMap<String, String>>()
+        .collect()
-}
-
-/// Replaces the commonly used services by mirrors, if asked to
-pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<String, String>) {
-    for val in urls.values_mut() {
-        for (k, v) in alts {
-            *val = val.replace(&format!("/{}/", k), &format!("/{}/", v));
-        }
-    }
-}
-
-/// Finds a Mastodon screen_name/base_url from a MastodonConfig
-pub fn find_mastodon_screen_name_by_twitter_screen_name(
-    twitter_screen_name: &str,
-    masto: &HashMap<String, MastodonConfig>,
-) -> Option<(String, String)> {
-    masto.iter().find_map(|(_, v)| {
-        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
-            Some((
-                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
-                v.base.to_owned(),
-            ))
-        } else {
-            None
-        }
-    })
-}
-
-/// Replaces the original quoted tweet by the corresponding toot
-pub fn replace_tweet_by_toot(
-    urls: &mut HashMap<String, String>,
-    twitter_screen_name: &str,
-    tweet_id: u64,
-    mastodon_screen_name: &str,
-    base_url: &str,
-    toot_id: &str,
-) {
-    for val in urls.values_mut() {
-        if val.to_lowercase().starts_with(&format!(
-            "https://twitter.com/{}/status/{}",
-            twitter_screen_name.to_lowercase(),
-            tweet_id
-        )) {
-            *val = format!("{}/@{}/{}", base_url, mastodon_screen_name, toot_id);
-        }
-    }
 }
 
 /// Gets Mastodon Data
 pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
+    let data = Data {
+        base: Cow::from(masto.base.to_owned()),
+        client_id: Cow::from(masto.client_id.to_owned()),
+        client_secret: Cow::from(masto.client_secret.to_owned()),
+        redirect: Cow::from(masto.redirect.to_owned()),
+        token: Cow::from(masto.token.to_owned()),
+    };
+
+    Mastodon::from(data)
+}
+
+/// Builds toot text from tweet
+pub fn build_basic_status(
+    tweet: &Tweet,
+    mentions: &HashMap<String, String>,
+    url_regex_filter: &Option<Regex>,
+    url_alt_services: &Option<HashMap<String, String>>,
+) -> String {
+    let mut toot = tweet.text.to_owned();
+
+    for decoded_url in decode_urls(&tweet.entities.urls, url_regex_filter, url_alt_services) {
+        toot = toot.replace(&decoded_url.0, &decoded_url.1);
+    }
+
+    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions)
+        .into_iter()
+        .chain(mentions.to_owned())
+        .collect::<HashMap<String, String>>()
+    {
+        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
+    }
+
+    decode_html_entities(&toot).to_string()
 }
 
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
-#[tokio::main]
+pub fn register(host: &str, screen_name: &str) {
-pub async fn register(host: &str, screen_name: &str) {
+    let mut builder = App::builder();
-    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
+    builder
+        .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
+        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
+        .scopes(
+            Scopes::write(Write::Accounts)
+                .and(Scopes::write(Write::Media))
+                .and(Scopes::write(Write::Statuses))
+                .and(Scopes::read(Read::Accounts)),
+        )
+        .website(Cow::from(
+            "https://framagit.org/veretcle/scootaloo".to_string(),
+        ));
 
-    let options = AppInputOptions {
+    let app = builder.build().expect("Cannot build the app");
-        redirect_uris: None,
-        scopes: Some(
-            [
-                "read:accounts".to_string(),
-                "write:accounts".to_string(),
-                "write:media".to_string(),
-                "write:statuses".to_string(),
-            ]
-            .to_vec(),
-        ),
-        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
-    };
 
-    let app_data = mastodon
+    let registration = Registration::new(host)
-        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
+        .register(app)
-        .await
+        .expect("Cannot build registration object");
-        .expect("Cannot build registration object!");
+    let url = registration
-
+        .authorize_url()
-    let url = app_data.url.expect("Cannot generate registration URI!");
+        .expect("Cannot generate registration URI!");
 
     println!("Click this link to authorize on Mastodon: {}", url);
     println!("Paste the returned authorization code: ");
@@ -159,47 +127,27 @@ pub async fn register(host: &str, screen_name: &str) {
         .read_line(&mut input)
         .expect("Unable to read back registration code!");
 
-    let token_data = mastodon
+    let code = input.trim();
-        .fetch_access_token(
+    let mastodon = registration
-            app_data.client_id.to_owned(),
+        .complete(code)
-            app_data.client_secret.to_owned(),
-            input.trim().to_string(),
-            megalodon::default::NO_REDIRECT.to_string(),
-        )
-        .await
         .expect("Unable to create access token!");
 
-    let mastodon = generator(
+    let toml = toml::to_string(&*mastodon).unwrap();
-        megalodon::SNS::Mastodon,
-        host.to_string(),
-        Some(token_data.access_token.to_owned()),
-        None,
-    );
 
     let current_account = mastodon
-        .verify_account_credentials()
+        .verify_credentials()
-        .await
+        .expect("Unable to access account information!");
-        .expect("Unable to access account information!")
-        .json();
 
     println!(
-        r#"Please insert the following block at the end of your configuration file:
+        "Please insert the following block at the end of your configuration file:
 [mastodon.{}]
-twitter_screen_name = "{}"
+twitter_screen_name = \"{}\"
-mastodon_screen_name = "{}"
+mastodon_screen_name = \"{}\"
-base = "{}"
+{}",
-client_id = "{}"
-client_secret = "{}"
-redirect = "{}"
-token = "{}""#,
         screen_name.to_lowercase(),
         screen_name,
         current_account.username,
-        host,
+        toml
-        app_data.client_id,
-        app_data.client_secret,
-        app_data.redirect_uri,
-        token_data.access_token,
     );
 }
 
@@ -207,209 +155,74 @@ token = "{}""#,
 mod tests {
     use super::*;
 
-    #[test]
+    use chrono::prelude::*;
-    fn test_replace_tweet_by_toot() {
+    use egg_mode::tweet::TweetEntities;
-        let mut associated_urls = HashMap::from([
-            (
-                "https://t.co/perdudeouf".to_string(),
-                "https://www.perdu.com".to_string(),
-            ),
-            (
-                "https://t.co/realquoteshere".to_string(),
-                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
-            ),
-            (
-                "https://t.co/almostthere".to_string(),
-                "https://twitter.com/NintendojoFR/status/nope".to_string(),
-            ),
-            (
-                "http://t.co/yetanotherone".to_string(),
-                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
-            ),
-        ]);
-
-        let expected_urls = HashMap::from([
-            (
-                "https://t.co/perdudeouf".to_string(),
-                "https://www.perdu.com".to_string(),
-            ),
-            (
-                "https://t.co/realquoteshere".to_string(),
-                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
-            ),
-            (
-                "https://t.co/almostthere".to_string(),
-                "https://twitter.com/NintendojoFR/status/nope".to_string(),
-            ),
-            (
-                "http://t.co/yetanotherone".to_string(),
-                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
-            ),
-        ]);
-
-        replace_tweet_by_toot(
-            &mut associated_urls,
-            "NintendojoFR",
-            1590047921633755136,
-            "nintendojofr",
-            "https://m.nintendojo.fr",
-            "109309605486908797",
-        );
-
-        assert_eq!(associated_urls, expected_urls);
-    }
-
-    #[test]
-    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
-        let masto_config = HashMap::from([
-            (
-                "test".to_string(),
-                MastodonConfig {
-                    twitter_screen_name: "tonpere".to_string(),
-                    mastodon_screen_name: Some("lalali".to_string()),
-                    twitter_page_size: None,
-                    base: "https://mstdn.net".to_string(),
-                    client_id: "".to_string(),
-                    client_secret: "".to_string(),
-                    redirect: "".to_string(),
-                    token: "".to_string(),
-                },
-            ),
-            (
-                "test2".to_string(),
-                MastodonConfig {
-                    twitter_screen_name: "tamerelol".to_string(),
-                    mastodon_screen_name: None,
-                    twitter_page_size: None,
-                    base: "https://mastoot.fr".to_string(),
-                    client_id: "".to_string(),
-                    client_secret: "".to_string(),
-                    redirect: "".to_string(),
-                    token: "".to_string(),
-                },
-            ),
-            (
-                "test3".to_string(),
-                MastodonConfig {
-                    twitter_screen_name: "NintendojoFR".to_string(),
-                    mastodon_screen_name: Some("nintendojofr".to_string()),
-                    twitter_page_size: None,
-                    base: "https://m.nintendojo.fr".to_string(),
-                    client_id: "".to_string(),
-                    client_secret: "".to_string(),
-                    redirect: "".to_string(),
-                    token: "".to_string(),
-                },
-            ),
-        ]);
-
-        // case sensitiveness, to avoid any mistake
-        assert_eq!(
-            None,
-            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
-        );
-        assert_eq!(
-            Some((
-                "nintendojofr".to_string(),
-                "https://m.nintendojo.fr".to_string()
-            )),
-            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
-        );
-        // should return None if twitter_screen_name is undefined
-        assert_eq!(
-            None,
-            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
-        );
-        assert_eq!(
-            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
-            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
-        );
-    }
 
     #[test]
     fn test_twitter_mentions() {
-        let mention_entities = vec![
+        let mention_entity = MentionEntity {
-            MentionEntity {
             id: 12345,
             range: (1, 3),
             name: "Ta Mere l0l".to_string(),
             screen_name: "tamerelol".to_string(),
-            },
+        };
-            MentionEntity {
-                id: 6789,
-                range: (1, 3),
-                name: "TONPERE".to_string(),
-                screen_name: "tonpere".to_string(),
-            },
-        ];
 
-        let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
+        let twitter_ums = vec![mention_entity];
 
-        let masto_config = HashMap::from([(
+        let mut expected_mentions = HashMap::new();
-            "test".to_string(),
+        expected_mentions.insert(
-            (MastodonConfig {
+            "@tamerelol".to_string(),
-                twitter_screen_name: "tonpere".to_string(),
+            "@tamerelol@twitter.com".to_string(),
-                mastodon_screen_name: Some("lalali".to_string()),
+        );
-                twitter_page_size: None,
-                base: "https://mstdn.net".to_string(),
-                client_id: "".to_string(),
-                client_secret: "".to_string(),
-                redirect: "".to_string(),
-                token: "".to_string(),
-            }),
-        )]);
 
-        twitter_mentions(&mut toot, &mention_entities, &masto_config);
+        let decoded_mentions = twitter_mentions(&twitter_ums);
 
-        assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
+        assert_eq!(expected_mentions, decoded_mentions);
     }
 
     #[test]
     fn test_decode_urls() {
-        let urls = HashMap::from([
+        let url_entity1 = UrlEntity {
-            (
-                "https://t.co/thisisatest".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
-            ),
-            (
-                "https://t.co/nopenotinclusive".to_string(),
-                "invité.es".to_string(),
-            ),
-        ]);
-
-        let mut toot =
-            "Rendez-vous sur https://t.co/thisisatest avec nos https://t.co/nopenotinclusive !"
-                .to_string();
-
-        decode_urls(&mut toot, &urls);
-
-        assert_eq!(
-            &toot,
-            "Rendez-vous sur https://www.nintendojo.fr/dojobar avec nos invité.es !"
-        );
-    }
-
-    #[test]
-    fn test_associate_urls() {
-        let urls = vec![
-            UrlEntity {
             display_url: "tamerelol".to_string(),
             expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
             range: (1, 3),
             url: "https://t.me/tamerelol".to_string(),
-            },
+        };
-            UrlEntity {
+
-                display_url: "sadcat".to_string(),
+        let url_entity2 = UrlEntity {
+            display_url: "tamerelol".to_string(),
             expanded_url: None,
             range: (1, 3),
-                url: "https://t.me/sadcat".to_string(),
+            url: "https://t.me/tamerelol".to_string(),
-            },
+        };
-            UrlEntity {
+
+        let wrong_url_entity = UrlEntity {
             display_url: "invité.es".to_string(),
             expanded_url: Some("http://xn--invit-fsa.es".to_string()),
             range: (85, 108),
             url: "https://t.co/WAUgnpHLmo".to_string(),
-            },
+        };
+
+        let rewritten_url_entity = UrlEntity {
+            display_url: "youtu.be/w5TrSaoYmZ8".to_string(),
+            expanded_url: Some("https://youtu.be/w5TrSaoYmZ8".to_string()),
+            range: (0, 23),
+            url: "https://t.co/fUVYXuF7tg".to_string(),
+        };
+
+        let re = Regex::new("(.+)\\.es$").ok();
+
+        let alt: HashMap<String, String> = HashMap::from([
+            ("youtube.com".to_string(), "invidio.us".to_string()),
+            ("youtu.be".to_string(), "invidio.us".to_string()),
+            ("www.youtube.com".to_string(), "invidio.us".to_string()),
+        ]);
+
+        let twitter_urls = vec![
+            url_entity1,
+            url_entity2,
+            wrong_url_entity,
+            rewritten_url_entity,
         ];
 
         let expected_urls = HashMap::from([
@@ -421,72 +234,83 @@ mod tests {
                 "https://t.co/WAUgnpHLmo".to_string(),
                 "invité.es".to_string(),
             ),
+            (
+                "https://t.co/fUVYXuF7tg".to_string(),
+                "https://invidio.us/w5TrSaoYmZ8".to_string(),
+            ),
         ]);
 
-        let re = Regex::new("(.+)\\.es$").ok();
+        let decoded_urls = decode_urls(&twitter_urls, &re, &Some(alt));
 
-        let associated_urls = associate_urls(&urls, &re);
+        assert_eq!(expected_urls, decoded_urls);
-
-        assert_eq!(associated_urls, expected_urls);
     }
 
     #[test]
-    fn test_replace_alt_services() {
+    fn test_build_basic_status() {
-        let mut associated_urls = HashMap::from([
+        let t = Tweet {
-            (
+            coordinates: None,
-                "https://t.co/youplaboom".to_string(),
+            created_at: Utc::now(),
-                "https://www.youtube.com/watch?v=dQw4w9WgXcQ".to_string(),
+            current_user_retweet: None,
-            ),
+            display_text_range: None,
-            (
+            entities: TweetEntities {
-                "https://t.co/thisisfine".to_string(),
+                hashtags: vec![],
-                "https://twitter.com/Nintendo/status/1594590628771688448".to_string(),
+                symbols: vec![],
-            ),
+                urls: vec![
-            (
+                        UrlEntity {
-                "https://t.co/nopenope".to_string(),
+                            display_url: "youtube.com/watch?v=w5TrSa…".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
+                            expanded_url: Some("https://www.youtube.com/watch?v=w5TrSaoYmZ8".to_string()),
-            ),
+                            range: (93, 116),
-            (
+                            url: "https://t.co/zXw0FfX2Nt".to_string(),
-                "https://t.co/broken".to_string(),
+                        }
-                "http://youtu.be".to_string(),
+                ],
-            ),
+                user_mentions: vec![
-            (
+                        MentionEntity {
-                "https://t.co/alsobroken".to_string(),
+                            id: 491500016,
-                "https://youtube.com".to_string(),
+                            range: (80, 95),
-            ),
+                            name: "Nintendo France".to_string(),
-        ]);
+                            screen_name: "NintendoFrance".to_string(),
+                        },
+                        MentionEntity {
+                            id: 999999999,
+                            range: (80, 95),
+                            name: "Willy Wonka".to_string(),
+                            screen_name: "WillyWonka".to_string(),
+                        },
+                    ],
+                media: None,
+            },
+            extended_entities: None,
+            favorite_count: 0,
+            favorited: None,
+            filter_level: None,
+            id: 1491541246984306693,
+            in_reply_to_user_id: None,
+            in_reply_to_screen_name: None,
+            in_reply_to_status_id: None,
+            lang: None,
+            place: None,
+            possibly_sensitive: None,
+            quoted_status: None,
+            quoted_status_id: None,
+            retweet_count: 0,
+            retweeted: None,
+            retweeted_status: None,
+            source: None,
+            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance @WillyWonka https://t.co/zXw0FfX2Nt".to_string(),
+            truncated: false,
+            user: None,
+            withheld_copyright: false,
+            withheld_in_countries: None,
+            withheld_scope: None,
+        };
 
-        let alt_services = HashMap::from([
+        let s: HashMap<String, String> = HashMap::from([(
-            ("twitter.com".to_string(), "nitter.net".to_string()),
+            "@WillyWonka".to_string(),
-            ("youtu.be".to_string(), "invidio.us".to_string()),
+            "@WillyWonka@chocolatefactory.org".to_string(),
-            ("www.youtube.com".to_string(), "invidio.us".to_string()),
+        )]);
-            ("youtube.com".to_string(), "invidio.us".to_string()),
-        ]);
 
-        let expected_urls = HashMap::from([
+        let t_out = build_basic_status(&t, &s, &None, &None);
-            (
-                "https://t.co/youplaboom".to_string(),
-                "https://invidio.us/watch?v=dQw4w9WgXcQ".to_string(),
-            ),
-            (
-                "https://t.co/thisisfine".to_string(),
-                "https://nitter.net/Nintendo/status/1594590628771688448".to_string(),
-            ),
-            (
-                "https://t.co/nopenope".to_string(),
-                "https://www.nintendojo.fr/dojobar".to_string(),
-            ),
-            (
-                "https://t.co/broken".to_string(),
-                "http://youtu.be".to_string(),
-            ),
-            (
-                "https://t.co/alsobroken".to_string(),
-                "https://youtube.com".to_string(),
-            ),
-        ]);
 
-        replace_alt_services(&mut associated_urls, &alt_services);
+        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com @WillyWonka@chocolatefactory.org https://www.youtube.com/watch?v=w5TrSaoYmZ8");
-
-        assert_eq!(associated_urls, expected_urls);
     }
 }

src/util.rs

@@ -1,16 +1,22 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
 
+use std::{borrow::Cow, error::Error};
+
 use egg_mode::tweet::Tweet;
-use futures::{stream, stream::StreamExt};
+
+use elefren::prelude::*;
+
 use log::{error, info, warn};
-use megalodon::{mastodon::Mastodon, megalodon::Megalodon};
+
 use reqwest::Url;
-use std::error::Error;
+
 use tokio::{
     fs::{create_dir_all, remove_file, File},
     io::copy,
 };
 
+use futures::{stream, stream::StreamExt};
+
 /// Generate associative table between media ids and tweet extended entities
 pub async fn generate_media_ids(
     tweet: &Tweet,
@@ -40,10 +46,8 @@ pub async fn generate_media_ids(
                     let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
 
                     // upload media to Mastodon
-                    let mastodon_media = mastodon
+                    let mastodon_media =
-                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        mastodon.media(Cow::from(local_tweet_media_path.to_owned()))?;
-                        .await?
-                        .json();
                     // at this point, we can safely erase the original file
                     // it doesn’t matter if we can’t remove, cache_media fn is idempotent
                     remove_file(&local_tweet_media_path).await.ok();