Merge branch '12-remove-elefren-deprecated-in-favor-of-megalodon-rs' into 'master'

Remove elefren (deprecated) in favor of megalodon-rs

Closes #12

See merge request veretcle/scootaloo!47

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scootaloo"
-version = "0.11.1"
+version = "1.0.0"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
@@ -13,9 +13,10 @@ toml = "^0.5"
 clap = "^4"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-tokio = { version = "^1", features = ["full"]}
+isolang = "^2"
+tokio = { version = "^1", features = ["rt"]}
 futures = "^0.3"
-elefren = "^0.22"
+megalodon = "^0.2"
 html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
@@ -24,3 +25,4 @@ mime = "^0.3"
 
 [profile.release]
 strip = true
+lto = true

src/error.rs

@@ -5,7 +5,7 @@ use std::{
     fmt::{Display, Formatter, Result},
 };
 
-use elefren::Error as elefrenError;
+use megalodon::error::Error as megalodonError;
 
 #[derive(Debug)]
 pub struct ScootalooError {
@@ -34,8 +34,8 @@ impl From<Box<dyn Error>> for ScootalooError {
     }
 }
 
-impl From<elefrenError> for ScootalooError {
+impl From<megalodonError> for ScootalooError {
-    fn from(error: elefrenError) -> Self {
+    fn from(error: megalodonError) -> Self {
-        ScootalooError::new(&format!("Error in elefren crate: {}", error))
+        ScootalooError::new(&format!("Error in megalodon crate: {}", error))
     }
 }

src/lib.rs

@@ -19,10 +19,11 @@ mod state;
 pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
-use elefren::{prelude::*, status_builder::StatusBuilder, Language};
 use futures::StreamExt;
 use html_escape::decode_html_entities;
+use isolang::Language;
 use log::info;
+use megalodon::{megalodon::PostStatusInputOptions, Megalodon};
 use regex::Regex;
 use rusqlite::Connection;
 use std::sync::Arc;
@@ -112,6 +113,45 @@ pub async fn run(config: Config) {
                         let mut associated_urls =
                             associate_urls(&tweet.entities.urls, &display_url_re);
 
+                        if let Some(q) = &tweet.quoted_status {
+                            if let Some(u) = &q.user {
+                                info!(
+                                    "Tweet {} contains a quote, we try to find it within the DB",
+                                    tweet.id
+                                );
+                                // we know we have a quote and a user, we can lock both the
+                                // connection to DB and global_config
+                                // we will release them manually as soon as they’re useless
+                                let lconn = task_conn.lock().await;
+                                let global_mastodon_config = global_mastodon_config.lock().await;
+                                if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
+                                {
+                                    info!("We have found the associated toot({})", &r.toot_id);
+                                    // drop conn immediately after the request: we won’t need it
+                                    // any more and the treatment there might be time-consuming
+                                    drop(lconn);
+                                    if let Some((m, t)) =
+                                        find_mastodon_screen_name_by_twitter_screen_name(
+                                            &r.twitter_screen_name,
+                                            &global_mastodon_config,
+                                        )
+                                    {
+                                        // drop the global conf, we have all we required, no need
+                                        // to block it further
+                                        drop(global_mastodon_config);
+                                        replace_tweet_by_toot(
+                                            &mut associated_urls,
+                                            &r.twitter_screen_name,
+                                            q.id,
+                                            &m,
+                                            &t,
+                                            &r.toot_id,
+                                        );
+                                    }
+                                }
+                            }
+                        }
+
                         if let Some(a) = &scootaloo_alt_services {
                             replace_alt_services(&mut associated_urls, a);
                         }
@@ -131,9 +171,21 @@ pub async fn run(config: Config) {
 
                     info!("Building corresponding Mastodon status");
 
-                    let mut status_builder = StatusBuilder::new();
+                    let mut post_status = PostStatusInputOptions {
+                        media_ids: None,
+                        poll: None,
+                        in_reply_to_id: None,
+                        sensitive: None,
+                        spoiler_text: None,
+                        visibility: None,
+                        scheduled_at: None,
+                        language: None,
+                        quote_id: None,
+                    };
 
-                    status_builder.status(status_text).media_ids(status_medias);
+                    if !status_medias.is_empty() {
+                        post_status.media_ids = Some(status_medias);
+                    }
 
                     // thread if necessary
                     if tweet.in_reply_to_user_id.is_some() {
@@ -143,7 +195,7 @@ pub async fn run(config: Config) {
                             &mastodon_config.twitter_screen_name,
                             tweet.in_reply_to_status_id,
                         ) {
-                            status_builder.in_reply_to(&r.toot_id);
+                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
                         }
                         drop(lconn);
                     }
@@ -151,16 +203,17 @@ pub async fn run(config: Config) {
                     // language if any
                     if let Some(l) = &tweet.lang {
                         if let Some(r) = Language::from_639_1(l) {
-                            status_builder.language(r);
+                            post_status.language = Some(r.to_string());
                         }
                     }
 
                     // can be activated for test purposes
-                    // status_builder.visibility(elefren::status_builder::Visibility::Private);
+                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
 
-                    let status = status_builder.build()?;
+                    let published_status = mastodon
-
+                        .post_status(status_text, Some(&post_status))
-                    let published_status = mastodon.new_status(status)?;
+                        .await?
+                        .json();
                     // this will return if it cannot publish the status preventing the last_tweet from
                     // being written into db
 

src/mastodon.rs

@@ -1,9 +1,9 @@
 use crate::config::MastodonConfig;
 
 use egg_mode::entities::{MentionEntity, UrlEntity};
-use elefren::{apps::App, prelude::*, scopes::Read, scopes::Scopes, scopes::Write};
+use megalodon::{generator, mastodon::Mastodon, megalodon::AppInputOptions};
 use regex::Regex;
-use std::{borrow::Cow, collections::HashMap, io::stdin};
+use std::{collections::HashMap, io::stdin};
 
 /// Decodes the Twitter mention to something that will make sense once Twitter has joined the
 /// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
@@ -81,45 +81,75 @@ pub fn replace_alt_services(urls: &mut HashMap<String, String>, alts: &HashMap<S
     }
 }
 
+/// Finds a Mastodon screen_name/base_url from a MastodonConfig
+pub fn find_mastodon_screen_name_by_twitter_screen_name(
+    twitter_screen_name: &str,
+    masto: &HashMap<String, MastodonConfig>,
+) -> Option<(String, String)> {
+    masto.iter().find_map(|(_, v)| {
+        if twitter_screen_name == v.twitter_screen_name && v.mastodon_screen_name.is_some() {
+            Some((
+                v.mastodon_screen_name.as_ref().unwrap().to_owned(),
+                v.base.to_owned(),
+            ))
+        } else {
+            None
+        }
+    })
+}
+
+/// Replaces the original quoted tweet by the corresponding toot
+pub fn replace_tweet_by_toot(
+    urls: &mut HashMap<String, String>,
+    twitter_screen_name: &str,
+    tweet_id: u64,
+    mastodon_screen_name: &str,
+    base_url: &str,
+    toot_id: &str,
+) {
+    for val in urls.values_mut() {
+        if val.to_lowercase().starts_with(&format!(
+            "https://twitter.com/{}/status/{}",
+            twitter_screen_name.to_lowercase(),
+            tweet_id
+        )) {
+            *val = format!("{}/@{}/{}", base_url, mastodon_screen_name, toot_id);
+        }
+    }
+}
+
 /// Gets Mastodon Data
 pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
+    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
-        base: Cow::from(masto.base.to_owned()),
-        client_id: Cow::from(masto.client_id.to_owned()),
-        client_secret: Cow::from(masto.client_secret.to_owned()),
-        redirect: Cow::from(masto.redirect.to_owned()),
-        token: Cow::from(masto.token.to_owned()),
-    };
-
-    Mastodon::from(data)
 }
 
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str, screen_name: &str) {
+#[tokio::main]
-    let mut builder = App::builder();
+pub async fn register(host: &str, screen_name: &str) {
-    builder
+    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
-        .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
-        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
-        .scopes(
-            Scopes::write(Write::Accounts)
-                .and(Scopes::write(Write::Media))
-                .and(Scopes::write(Write::Statuses))
-                .and(Scopes::read(Read::Accounts)),
-        )
-        .website(Cow::from(
-            "https://framagit.org/veretcle/scootaloo".to_string(),
-        ));
 
-    let app = builder.build().expect("Cannot build the app");
+    let options = AppInputOptions {
+        redirect_uris: None,
+        scopes: Some(
+            [
+                "read:accounts".to_string(),
+                "write:accounts".to_string(),
+                "write:media".to_string(),
+                "write:statuses".to_string(),
+            ]
+            .to_vec(),
+        ),
+        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
+    };
 
-    let registration = Registration::new(host)
+    let app_data = mastodon
-        .register(app)
+        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
-        .expect("Cannot build registration object");
+        .await
-    let url = registration
+        .expect("Cannot build registration object!");
-        .authorize_url()
+
-        .expect("Cannot generate registration URI!");
+    let url = app_data.url.expect("Cannot generate registration URI!");
 
     println!("Click this link to authorize on Mastodon: {}", url);
     println!("Paste the returned authorization code: ");
@@ -129,27 +159,47 @@ pub fn register(host: &str, screen_name: &str) {
         .read_line(&mut input)
         .expect("Unable to read back registration code!");
 
-    let code = input.trim();
+    let token_data = mastodon
-    let mastodon = registration
+        .fetch_access_token(
-        .complete(code)
+            app_data.client_id.to_owned(),
+            app_data.client_secret.to_owned(),
+            input.trim().to_string(),
+            megalodon::default::NO_REDIRECT.to_string(),
+        )
+        .await
         .expect("Unable to create access token!");
 
-    let toml = toml::to_string(&*mastodon).unwrap();
+    let mastodon = generator(
+        megalodon::SNS::Mastodon,
+        host.to_string(),
+        Some(token_data.access_token.to_owned()),
+        None,
+    );
 
     let current_account = mastodon
-        .verify_credentials()
+        .verify_account_credentials()
-        .expect("Unable to access account information!");
+        .await
+        .expect("Unable to access account information!")
+        .json();
 
     println!(
-        "Please insert the following block at the end of your configuration file:
+        r#"Please insert the following block at the end of your configuration file:
 [mastodon.{}]
-twitter_screen_name = \"{}\"
+twitter_screen_name = "{}"
-mastodon_screen_name = \"{}\"
+mastodon_screen_name = "{}"
-{}",
+base = "{}"
+client_id = "{}"
+client_secret = "{}"
+redirect = "{}"
+token = "{}""#,
         screen_name.to_lowercase(),
         screen_name,
         current_account.username,
-        toml
+        host,
+        app_data.client_id,
+        app_data.client_secret,
+        app_data.redirect_uri,
+        token_data.access_token,
     );
 }
 
@@ -157,6 +207,125 @@ mastodon_screen_name = \"{}\"
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_replace_tweet_by_toot() {
+        let mut associated_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://twitter.com/nintendojofr/status/1590047921633755136".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://twitter.com/NINTENDOJOFR/status/1590047921633755136".to_string(),
+            ),
+        ]);
+
+        let expected_urls = HashMap::from([
+            (
+                "https://t.co/perdudeouf".to_string(),
+                "https://www.perdu.com".to_string(),
+            ),
+            (
+                "https://t.co/realquoteshere".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+            (
+                "https://t.co/almostthere".to_string(),
+                "https://twitter.com/NintendojoFR/status/nope".to_string(),
+            ),
+            (
+                "http://t.co/yetanotherone".to_string(),
+                "https://m.nintendojo.fr/@nintendojofr/109309605486908797".to_string(),
+            ),
+        ]);
+
+        replace_tweet_by_toot(
+            &mut associated_urls,
+            "NintendojoFR",
+            1590047921633755136,
+            "nintendojofr",
+            "https://m.nintendojo.fr",
+            "109309605486908797",
+        );
+
+        assert_eq!(associated_urls, expected_urls);
+    }
+
+    #[test]
+    fn test_find_mastodon_screen_name_by_twitter_screen_name() {
+        let masto_config = HashMap::from([
+            (
+                "test".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tonpere".to_string(),
+                    mastodon_screen_name: Some("lalali".to_string()),
+                    twitter_page_size: None,
+                    base: "https://mstdn.net".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test2".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "tamerelol".to_string(),
+                    mastodon_screen_name: None,
+                    twitter_page_size: None,
+                    base: "https://mastoot.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+            (
+                "test3".to_string(),
+                MastodonConfig {
+                    twitter_screen_name: "NintendojoFR".to_string(),
+                    mastodon_screen_name: Some("nintendojofr".to_string()),
+                    twitter_page_size: None,
+                    base: "https://m.nintendojo.fr".to_string(),
+                    client_id: "".to_string(),
+                    client_secret: "".to_string(),
+                    redirect: "".to_string(),
+                    token: "".to_string(),
+                },
+            ),
+        ]);
+
+        // case sensitiveness, to avoid any mistake
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("nintendojofr", &masto_config)
+        );
+        assert_eq!(
+            Some((
+                "nintendojofr".to_string(),
+                "https://m.nintendojo.fr".to_string()
+            )),
+            find_mastodon_screen_name_by_twitter_screen_name("NintendojoFR", &masto_config)
+        );
+        // should return None if twitter_screen_name is undefined
+        assert_eq!(
+            None,
+            find_mastodon_screen_name_by_twitter_screen_name("tamerelol", &masto_config)
+        );
+        assert_eq!(
+            Some(("lalali".to_string(), "https://mstdn.net".to_string())),
+            find_mastodon_screen_name_by_twitter_screen_name("tonpere", &masto_config)
+        );
+    }
+
     #[test]
     fn test_twitter_mentions() {
         let mention_entities = vec![
@@ -176,7 +345,7 @@ mod tests {
 
         let mut toot = ":kikoo: @tamerelol @tonpere !".to_string();
 
-        let scootaloo_config = HashMap::from([(
+        let masto_config = HashMap::from([(
             "test".to_string(),
             (MastodonConfig {
                 twitter_screen_name: "tonpere".to_string(),
@@ -190,7 +359,7 @@ mod tests {
             }),
         )]);
 
-        twitter_mentions(&mut toot, &mention_entities, &scootaloo_config);
+        twitter_mentions(&mut toot, &mention_entities, &masto_config);
 
         assert_eq!(&toot, ":kikoo: @tamerelol@twitter.com @lalali@mstdn.net !");
     }

src/util.rs

@@ -1,22 +1,16 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
 
-use std::{borrow::Cow, error::Error};
-
 use egg_mode::tweet::Tweet;
-
+use futures::{stream, stream::StreamExt};
-use elefren::prelude::*;
-
 use log::{error, info, warn};
-
+use megalodon::{mastodon::Mastodon, megalodon::Megalodon};
 use reqwest::Url;
-
+use std::error::Error;
 use tokio::{
     fs::{create_dir_all, remove_file, File},
     io::copy,
 };
 
-use futures::{stream, stream::StreamExt};
-
 /// Generate associative table between media ids and tweet extended entities
 pub async fn generate_media_ids(
     tweet: &Tweet,
@@ -46,8 +40,10 @@ pub async fn generate_media_ids(
                     let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
 
                     // upload media to Mastodon
-                    let mastodon_media =
+                    let mastodon_media = mastodon
-                        mastodon.media(Cow::from(local_tweet_media_path.to_owned()))?;
+                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        .await?
+                        .json();
                     // at this point, we can safely erase the original file
                     // it doesn’t matter if we can’t remove, cache_media fn is idempotent
                     remove_file(&local_tweet_media_path).await.ok();