Merge branch 'fix_attachment' into 'master'

Fix attachment

See merge request veretcle/scootaloo!52

Cargo.toml

@@ -1,21 +1,23 @@
 [package]
 name = "scootaloo"
-version = "0.12.2"
+version = "1.1.4"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+base64 = "^0.13"
 regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
 clap = "^4"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-tokio = { version = "^1", features = ["full"]}
+isolang = "^2"
+tokio = { version = "^1", features = ["rt"]}
 futures = "^0.3"
-elefren = "^0.22"
+megalodon = "^0.3.6"
 html-escape = "^0.2"
 reqwest = "^0.11"
 log = "^0.4"
@@ -24,3 +26,5 @@ mime = "^0.3"
 
 [profile.release]
 strip = true
+lto = true
+codegen-units = 1

README.md

@@ -83,22 +83,20 @@ You can then run the application via `cron` for example. Here is the generic usa
 ```sh
 A Twitter to Mastodon bot
 
-USAGE:
+Usage: scootaloo [OPTIONS] [COMMAND]
-    scootaloo [OPTIONS] [SUBCOMMAND]
 
-FLAGS:
+Commands:
-    -h, --help       Prints help information
+  register     Command to register to a Mastodon Instance
-    -V, --version    Prints version information
+  init         Command to init Scootaloo DB
+  migrate      Command to migrate Scootaloo DB
+  copyprofile  Command to copy a Twitter profile into Mastodon
+  help         Print this message or the help of the given subcommand(s)
 
-OPTIONS:
+Options:
-    -c, --config <CONFIG_FILE>    TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)
+  -c, --config <CONFIG_FILE>  TOML config file for scootaloo [default: /usr/local/etc/scootaloo.toml]
-    -l, --loglevel <LOGLEVEL>     Log level. Valid values are: Off, Warn, Error, Info, Debug
+  -l, --loglevel <LOGLEVEL>   Log level [possible values: Off, Warn, Error, Info, Debug]
-
+  -h, --help                  Print help information
-SUBCOMMANDS:
+  -V, --version               Print version information
-    help        Prints this message or the help of the given subcommand(s)
-    init        Command to init Scootaloo DB
-    migrate     Command to migrate Scootaloo DB
-    register    Command to register to a Mastodon Instance
 ```
 
 # Quirks

src/error.rs

@@ -5,7 +5,7 @@ use std::{
     fmt::{Display, Formatter, Result},
 };
 
-use elefren::Error as elefrenError;
+use megalodon::error::Error as megalodonError;
 
 #[derive(Debug)]
 pub struct ScootalooError {
@@ -34,8 +34,8 @@ impl From<Box<dyn Error>> for ScootalooError {
     }
 }
 
-impl From<elefrenError> for ScootalooError {
+impl From<megalodonError> for ScootalooError {
-    fn from(error: elefrenError) -> Self {
+    fn from(error: megalodonError) -> Self {
-        ScootalooError::new(&format!("Error in elefren crate: {}", error))
+        ScootalooError::new(&format!("Error in megalodon crate: {}", error))
     }
 }

src/lib.rs

@@ -13,16 +13,19 @@ mod twitter;
 use twitter::*;
 
 mod util;
-use crate::util::generate_media_ids;
+use util::{base64_media, generate_media_ids};
 
 mod state;
 pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
-use elefren::{prelude::*, status_builder::StatusBuilder, Language};
 use futures::StreamExt;
 use html_escape::decode_html_entities;
+use isolang::Language;
 use log::info;
+use megalodon::{
+    megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
+};
 use regex::Regex;
 use rusqlite::Connection;
 use std::sync::Arc;
@@ -170,9 +173,21 @@ pub async fn run(config: Config) {
 
                     info!("Building corresponding Mastodon status");
 
-                    let mut status_builder = StatusBuilder::new();
+                    let mut post_status = PostStatusInputOptions {
+                        media_ids: None,
+                        poll: None,
+                        in_reply_to_id: None,
+                        sensitive: None,
+                        spoiler_text: None,
+                        visibility: None,
+                        scheduled_at: None,
+                        language: None,
+                        quote_id: None,
+                    };
 
-                    status_builder.status(status_text).media_ids(status_medias);
+                    if !status_medias.is_empty() {
+                        post_status.media_ids = Some(status_medias);
+                    }
 
                     // thread if necessary
                     if tweet.in_reply_to_user_id.is_some() {
@@ -182,7 +197,7 @@ pub async fn run(config: Config) {
                             &mastodon_config.twitter_screen_name,
                             tweet.in_reply_to_status_id,
                         ) {
-                            status_builder.in_reply_to(&r.toot_id);
+                            post_status.in_reply_to_id = Some(r.toot_id.to_owned());
                         }
                         drop(lconn);
                     }
@@ -190,16 +205,17 @@ pub async fn run(config: Config) {
                     // language if any
                     if let Some(l) = &tweet.lang {
                         if let Some(r) = Language::from_639_1(l) {
-                            status_builder.language(r);
+                            post_status.language = Some(r.to_string());
                         }
                     }
 
                     // can be activated for test purposes
-                    // status_builder.visibility(elefren::status_builder::Visibility::Private);
+                    // post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
 
-                    let status = status_builder.build()?;
+                    let published_status = mastodon
-
+                        .post_status(status_text, Some(&post_status))
-                    let published_status = mastodon.new_status(status)?;
+                        .await?
+                        .json();
                     // this will return if it cannot publish the status preventing the last_tweet from
                     // being written into db
 
@@ -228,3 +244,72 @@ pub async fn run(config: Config) {
         }
     }
 }
+
+/// Copies the Twitter profile into Mastodon
+#[tokio::main]
+pub async fn profile(config: Config, bot: Option<bool>) {
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            let token = get_oauth2_token(&config.twitter);
+
+            spawn(async move {
+                // get the user of the last tweet of the feed
+                let twitter_user =
+                    get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
+                        .await?
+                        .first()
+                        .ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
+                        .clone()
+                        .user
+                        .ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
+
+                let note = get_note_from_description(
+                    &twitter_user.description,
+                    &twitter_user.entities.description.urls,
+                );
+
+                let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
+
+                let display_name = Some(String::from_utf16_lossy(
+                    &twitter_user
+                        .name
+                        .encode_utf16()
+                        .take(30)
+                        .collect::<Vec<u16>>(),
+                ));
+
+                let header = match twitter_user.profile_banner_url {
+                    Some(h) => Some(base64_media(&h).await?),
+                    None => None,
+                };
+
+                let update_creds = UpdateCredentialsInputOptions {
+                    bot,
+                    display_name,
+                    note,
+                    avatar: Some(
+                        base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
+                            .await?,
+                    ),
+                    header,
+                    fields_attributes,
+                    ..Default::default()
+                };
+
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                mastodon.update_credentials(Some(&update_creds)).await?;
+
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {}", e),
+            Err(e) => eprintln!("Error with thread: {}", e),
+            _ => (),
+        }
+    }
+}

src/main.rs

@@ -1,4 +1,4 @@
-use clap::{Arg, Command};
+use clap::{Arg, ArgAction, Command};
 use log::LevelFilter;
 use scootaloo::*;
 use simple_logger::SimpleLogger;
@@ -63,7 +63,7 @@ fn main() {
                         .short('c')
                         .long("config")
                         .value_name("CONFIG_FILE")
-                        .help(&format!(
+                        .help(format!(
                             "TOML config file for scootaloo (default {})",
                             DEFAULT_CONFIG_PATH
                         ))
@@ -81,18 +81,49 @@ fn main() {
                     .short('c')
                     .long("config")
                     .value_name("CONFIG_FILE")
-                    .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
+                    .help(format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
                     .default_value(DEFAULT_CONFIG_PATH)
                     .num_args(1)
                     .display_order(1),
             )
             .arg(
-                    Arg::new("name")
+                Arg::new("name")
-                        .short('n')
+                    .short('n')
-                        .long("name")
+                    .long("name")
-                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                    .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .num_args(1)
+                    .num_args(1)
-                        .display_order(2)
+                    .display_order(2)
+            )
+        )
+        .subcommand(
+            Command::new("copyprofile")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to copy a Twitter profile into Mastodon")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help(format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Mastodon Config name (as seen in the config file)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+            .arg(
+                Arg::new("bot")
+                    .short('b')
+                    .long("bot")
+                    .help("Declare user as bot")
+                    .action(ArgAction::SetTrue)
+                    .display_order(3)
             )
         )
         .get_matches();
@@ -123,6 +154,22 @@ fn main() {
             .unwrap();
             return;
         }
+        Some(("copyprofile", sub_m)) => {
+            let mut config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            // filters out the user passed in cli from the global configuration
+            if let Some(m) = sub_m.get_one::<String>("name") {
+                match config.mastodon.get(m) {
+                    Some(_) => {
+                        config.mastodon.retain(|k, _| k == m);
+                    }
+                    None => panic!("Config file does not contain conf for {}", &m),
+                }
+            }
+
+            profile(config, sub_m.get_flag("bot").then_some(true));
+
+            return;
+        }
         _ => (),
     }
 

src/mastodon.rs

@@ -1,9 +1,16 @@
 use crate::config::MastodonConfig;
 
-use egg_mode::entities::{MentionEntity, UrlEntity};
+use egg_mode::{
-use elefren::{apps::App, prelude::*, scopes::Read, scopes::Scopes, scopes::Write};
+    entities::{MentionEntity, UrlEntity},
+    user::UserEntityDetail,
+};
+use megalodon::{
+    generator,
+    mastodon::Mastodon,
+    megalodon::{AppInputOptions, CredentialsFieldAttribute},
+};
 use regex::Regex;
-use std::{borrow::Cow, collections::HashMap, io::stdin};
+use std::{collections::HashMap, io::stdin};
 
 /// Decodes the Twitter mention to something that will make sense once Twitter has joined the
 /// Fediverse. Users in the global user list of Scootaloo are rewritten, as they are Mastodon users
@@ -120,43 +127,62 @@ pub fn replace_tweet_by_toot(
 
 /// Gets Mastodon Data
 pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
+    Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
-        base: Cow::from(masto.base.to_owned()),
+}
-        client_id: Cow::from(masto.client_id.to_owned()),
-        client_secret: Cow::from(masto.client_secret.to_owned()),
-        redirect: Cow::from(masto.redirect.to_owned()),
-        token: Cow::from(masto.token.to_owned()),
-    };
 
-    Mastodon::from(data)
+/// Gets note from twitter_user description
+pub fn get_note_from_description(t: &Option<String>, urls: &[UrlEntity]) -> Option<String> {
+    t.as_ref().map(|d| {
+        let mut n = d.to_owned();
+        let a_urls = associate_urls(urls, &None);
+        decode_urls(&mut n, &a_urls);
+        n
+    })
+}
+
+/// Gets fields_attribute from UserEntityDetail
+pub fn get_attribute_from_url(
+    user_entity_detail: &Option<UserEntityDetail>,
+) -> Option<Vec<CredentialsFieldAttribute>> {
+    user_entity_detail.as_ref().and_then(|u| {
+        u.urls.first().and_then(|v| {
+            v.expanded_url.as_ref().map(|e| {
+                vec![CredentialsFieldAttribute {
+                    name: v.display_url.to_string(),
+                    value: e.to_string(),
+                }]
+            })
+        })
+    })
 }
 
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str, screen_name: &str) {
+#[tokio::main]
-    let mut builder = App::builder();
+pub async fn register(host: &str, screen_name: &str) {
-    builder
+    let mastodon = generator(megalodon::SNS::Mastodon, host.to_string(), None, None);
-        .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
-        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
-        .scopes(
-            Scopes::write(Write::Accounts)
-                .and(Scopes::write(Write::Media))
-                .and(Scopes::write(Write::Statuses))
-                .and(Scopes::read(Read::Accounts)),
-        )
-        .website(Cow::from(
-            "https://framagit.org/veretcle/scootaloo".to_string(),
-        ));
 
-    let app = builder.build().expect("Cannot build the app");
+    let options = AppInputOptions {
+        redirect_uris: None,
+        scopes: Some(
+            [
+                "read:accounts".to_string(),
+                "write:accounts".to_string(),
+                "write:media".to_string(),
+                "write:statuses".to_string(),
+            ]
+            .to_vec(),
+        ),
+        website: Some("https://framagit.org/veretcle/scootaloo".to_string()),
+    };
 
-    let registration = Registration::new(host)
+    let app_data = mastodon
-        .register(app)
+        .register_app(env!("CARGO_PKG_NAME").to_string(), &options)
-        .expect("Cannot build registration object");
+        .await
-    let url = registration
+        .expect("Cannot build registration object!");
-        .authorize_url()
+
-        .expect("Cannot generate registration URI!");
+    let url = app_data.url.expect("Cannot generate registration URI!");
 
     println!("Click this link to authorize on Mastodon: {}", url);
     println!("Paste the returned authorization code: ");
@@ -166,27 +192,47 @@ pub fn register(host: &str, screen_name: &str) {
         .read_line(&mut input)
         .expect("Unable to read back registration code!");
 
-    let code = input.trim();
+    let token_data = mastodon
-    let mastodon = registration
+        .fetch_access_token(
-        .complete(code)
+            app_data.client_id.to_owned(),
+            app_data.client_secret.to_owned(),
+            input.trim().to_string(),
+            megalodon::default::NO_REDIRECT.to_string(),
+        )
+        .await
         .expect("Unable to create access token!");
 
-    let toml = toml::to_string(&*mastodon).unwrap();
+    let mastodon = generator(
+        megalodon::SNS::Mastodon,
+        host.to_string(),
+        Some(token_data.access_token.to_owned()),
+        None,
+    );
 
     let current_account = mastodon
-        .verify_credentials()
+        .verify_account_credentials()
-        .expect("Unable to access account information!");
+        .await
+        .expect("Unable to access account information!")
+        .json();
 
     println!(
-        "Please insert the following block at the end of your configuration file:
+        r#"Please insert the following block at the end of your configuration file:
 [mastodon.{}]
-twitter_screen_name = \"{}\"
+twitter_screen_name = "{}"
-mastodon_screen_name = \"{}\"
+mastodon_screen_name = "{}"
-{}",
+base = "{}"
+client_id = "{}"
+client_secret = "{}"
+redirect = "{}"
+token = "{}""#,
         screen_name.to_lowercase(),
         screen_name,
         current_account.username,
-        toml
+        host,
+        app_data.client_id,
+        app_data.client_secret,
+        app_data.redirect_uri,
+        token_data.access_token,
     );
 }
 
@@ -194,6 +240,63 @@ mastodon_screen_name = \"{}\"
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_get_attribute_from_url() {
+        let expected_credentials_field_attribute = CredentialsFieldAttribute {
+            name: "Nintendojo.fr".to_string(),
+            value: "https://www.nintendojo.fr".to_string(),
+        };
+
+        let true_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr".to_string()),
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        let false_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: None,
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        assert!(get_attribute_from_url(&None).is_none());
+        assert!(get_attribute_from_url(&Some(UserEntityDetail { urls: false_urls })).is_none());
+
+        let binding = get_attribute_from_url(&Some(UserEntityDetail { urls: true_urls })).unwrap();
+        let result_credentials_field_attribute = binding.first().unwrap();
+
+        assert_eq!(
+            result_credentials_field_attribute.name,
+            expected_credentials_field_attribute.name
+        );
+        assert_eq!(
+            result_credentials_field_attribute.value,
+            expected_credentials_field_attribute.value
+        );
+    }
+
+    #[test]
+    fn test_get_note_from_description() {
+        let urls = vec![UrlEntity {
+            display_url: "tamerelol".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+            range: (1, 3),
+            url: "https://t.me/tamerelol".to_string(),
+        }];
+
+        let some_description = Some("Youpi | https://t.me/tamerelol".to_string());
+        let none_description = None;
+
+        assert_eq!(
+            get_note_from_description(&some_description, &urls),
+            Some("Youpi | https://www.nintendojo.fr/dojobar".to_string())
+        );
+
+        assert_eq!(get_note_from_description(&none_description, &urls), None);
+    }
+
     #[test]
     fn test_replace_tweet_by_toot() {
         let mut associated_urls = HashMap::from([

src/util.rs

@@ -1,22 +1,22 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
 
-use std::{borrow::Cow, error::Error};
+use base64::encode;
-
 use egg_mode::tweet::Tweet;
-
+use futures::{stream, stream::StreamExt};
-use elefren::prelude::*;
-
 use log::{error, info, warn};
-
+use megalodon::{
+    entities::UploadMedia::{AsyncAttachment, Attachment},
+    error,
+    mastodon::Mastodon,
+    megalodon::Megalodon,
+};
 use reqwest::Url;
-
+use std::error::Error;
 use tokio::{
     fs::{create_dir_all, remove_file, File},
     io::copy,
 };
 
-use futures::{stream, stream::StreamExt};
-
 /// Generate associative table between media ids and tweet extended entities
 pub async fn generate_media_ids(
     tweet: &Tweet,
@@ -46,13 +46,20 @@ pub async fn generate_media_ids(
                     let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
 
                     // upload media to Mastodon
-                    let mastodon_media =
+                    let mastodon_media = mastodon
-                        mastodon.media(Cow::from(local_tweet_media_path.to_owned()))?;
+                        .upload_media(local_tweet_media_path.to_owned(), None)
+                        .await?
+                        .json();
                     // at this point, we can safely erase the original file
                     // it doesn’t matter if we can’t remove, cache_media fn is idempotent
                     remove_file(&local_tweet_media_path).await.ok();
 
-                    Ok::<String, ScootalooError>(mastodon_media.id)
+                    let id = match mastodon_media {
+                        Attachment(m) => m.id,
+                        AsyncAttachment(m) => wait_until_uploaded(&mastodon, &m.id).await?,
+                    };
+
+                    Ok::<String, ScootalooError>(id)
                 })
             })
             .buffered(4); // there are max four medias per tweet and they need to be treated in
@@ -75,6 +82,44 @@ pub async fn generate_media_ids(
     (media_url, media_ids)
 }
 
+/// Wait on uploaded medias when necessary
+async fn wait_until_uploaded(client: &Mastodon, id: &str) -> Result<String, error::Error> {
+    loop {
+        let res = client.get_media(id.to_string()).await;
+        return match res {
+            Ok(res) => Ok(res.json.id),
+            Err(err) => match err {
+                error::Error::OwnError(ref own_err) => match own_err.kind {
+                    error::Kind::HTTPPartialContentError => continue,
+                    _ => Err(err),
+                },
+                _ => Err(err),
+            },
+        };
+    }
+}
+
+/// Transforms the media into a base64 equivalent
+pub async fn base64_media(u: &str) -> Result<String, Box<dyn Error>> {
+    let mut response = reqwest::get(u).await?;
+
+    let mut buffer = Vec::new();
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut buffer).await?;
+    }
+
+    let content_type = response
+        .headers()
+        .get("content-type")
+        .ok_or_else(|| ScootalooError::new(&format!("Cannot get media content type for {}", u)))?
+        .to_str()?;
+
+    let encoded_f = encode(buffer);
+
+    Ok(format!("data:{};base64,{}", content_type, encoded_f))
+}
+
 /// Gets and caches Twitter Media inside the determined temp dir
 pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
     // create dir
@@ -133,4 +178,17 @@ mod tests {
 
         remove_dir_all(TMP_DIR).unwrap();
     }
+
+    #[tokio::test]
+    async fn test_base64_media() {
+        let img = base64_media(
+            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
+        )
+        .await
+        .unwrap();
+
+        assert!(img.starts_with("data:image/png;base64,"));
+
+        assert!(img.ends_with("="));
+    }
 }