Merge branch 'feat_links' into 'master'

feat: fields_attributes and note inside copyprofile

See merge request veretcle/scootaloo!50

Cargo.lock

@@ -1342,8 +1342,9 @@ dependencies = [
 
 [[package]]
 name = "scootaloo"
-version = "1.0.0"
+version = "1.1.2"
 dependencies = [
+ "base64",
  "clap",
  "egg-mode",
  "futures",

Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "scootaloo"
-version = "1.0.0"
+version = "1.1.2"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+base64 = "^0.13"
 regex = "^1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
@@ -26,3 +27,4 @@ mime = "^0.3"
 [profile.release]
 strip = true
 lto = true
+codegen-units = 1

src/lib.rs

@@ -13,7 +13,7 @@ mod twitter;
 use twitter::*;
 
 mod util;
-use crate::util::generate_media_ids;
+use util::{base64_media, generate_media_ids};
 
 mod state;
 pub use state::{init_db, migrate_db};
@@ -23,7 +23,9 @@ use futures::StreamExt;
 use html_escape::decode_html_entities;
 use isolang::Language;
 use log::info;
-use megalodon::{megalodon::PostStatusInputOptions, Megalodon};
+use megalodon::{
+    megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
+};
 use regex::Regex;
 use rusqlite::Connection;
 use std::sync::Arc;
@@ -242,3 +244,74 @@ pub async fn run(config: Config) {
         }
     }
 }
+
+/// Copies the Twitter profile into Mastodon
+#[tokio::main]
+pub async fn profile(config: Config, bot: Option<bool>) {
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            let token = get_oauth2_token(&config.twitter);
+
+            spawn(async move {
+                // get the user of the last tweet of the feed
+                let twitter_user =
+                    get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
+                        .await?
+                        .first()
+                        .ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
+                        .clone()
+                        .user
+                        .ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
+
+                let note = get_note_from_description(
+                    &twitter_user.description,
+                    &twitter_user.entities.description.urls,
+                );
+
+                // let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
+
+                let display_name = Some(String::from_utf16_lossy(
+                    &twitter_user
+                        .name
+                        .encode_utf16()
+                        .take(30)
+                        .collect::<Vec<u16>>(),
+                ));
+
+                let header = match twitter_user.profile_banner_url {
+                    Some(h) => Some(base64_media(&h).await?),
+                    None => None,
+                };
+
+                let update_creds = UpdateCredentialsInputOptions {
+                    discoverable: None,
+                    bot,
+                    display_name,
+                    note,
+                    avatar: Some(
+                        base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
+                            .await?,
+                    ),
+                    header,
+                    locked: None,
+                    source: None,
+                    fields_attributes: None,
+                };
+
+                let mastodon = get_mastodon_token(&mastodon_config);
+
+                mastodon.update_credentials(Some(&update_creds)).await?;
+
+                Ok::<(), ScootalooError>(())
+            })
+        })
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
+
+    while let Some(result) = stream.next().await {
+        match result {
+            Ok(Err(e)) => eprintln!("Error within thread: {}", e),
+            Err(e) => eprintln!("Error with thread: {}", e),
+            _ => (),
+        }
+    }
+}

src/main.rs

@@ -1,4 +1,4 @@
-use clap::{Arg, Command};
+use clap::{Arg, ArgAction, Command};
 use log::LevelFilter;
 use scootaloo::*;
 use simple_logger::SimpleLogger;
@@ -87,12 +87,43 @@ fn main() {
                     .display_order(1),
             )
             .arg(
-                    Arg::new("name")
+                Arg::new("name")
-                        .short('n')
+                    .short('n')
-                        .long("name")
+                    .long("name")
-                        .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
+                    .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .num_args(1)
+                    .num_args(1)
-                        .display_order(2)
+                    .display_order(2)
+            )
+        )
+        .subcommand(
+            Command::new("copyprofile")
+            .version(env!("CARGO_PKG_VERSION"))
+            .about("Command to copy a Twitter profile into Mastodon")
+            .arg(
+                Arg::new("config")
+                    .short('c')
+                    .long("config")
+                    .value_name("CONFIG_FILE")
+                    .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
+                    .display_order(1),
+            )
+            .arg(
+                Arg::new("name")
+                    .short('n')
+                    .long("name")
+                    .help("Mastodon Config name (as seen in the config file)")
+                    .num_args(1)
+                    .display_order(2)
+            )
+            .arg(
+                Arg::new("bot")
+                    .short('b')
+                    .long("bot")
+                    .help("Declare user as bot")
+                    .action(ArgAction::SetTrue)
+                    .display_order(3)
             )
         )
         .get_matches();
@@ -123,6 +154,22 @@ fn main() {
             .unwrap();
             return;
         }
+        Some(("copyprofile", sub_m)) => {
+            let mut config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+            // filters out the user passed in cli from the global configuration
+            if let Some(m) = sub_m.get_one::<String>("name") {
+                match config.mastodon.get(m) {
+                    Some(_) => {
+                        config.mastodon.retain(|k, _| k == m);
+                    }
+                    None => panic!("Config file does not contain conf for {}", &m),
+                }
+            }
+
+            profile(config, sub_m.get_flag("bot").then_some(true));
+
+            return;
+        }
         _ => (),
     }
 

src/mastodon.rs

@@ -1,7 +1,14 @@
 use crate::config::MastodonConfig;
 
-use egg_mode::entities::{MentionEntity, UrlEntity};
+use egg_mode::{
-use megalodon::{generator, mastodon::Mastodon, megalodon::AppInputOptions};
+    entities::{MentionEntity, UrlEntity},
+    user::UserEntityDetail,
+};
+use megalodon::{
+    generator,
+    mastodon::Mastodon,
+    megalodon::{AppInputOptions, CredentialsFieldAttribute},
+};
 use regex::Regex;
 use std::{collections::HashMap, io::stdin};
 
@@ -123,6 +130,33 @@ pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
     Mastodon::new(masto.base.to_string(), Some(masto.token.to_string()), None)
 }
 
+/// Gets note from twitter_user description
+pub fn get_note_from_description(t: &Option<String>, urls: &[UrlEntity]) -> Option<String> {
+    t.as_ref().map(|d| {
+        let mut n = d.to_owned();
+        let a_urls = associate_urls(urls, &None);
+        decode_urls(&mut n, &a_urls);
+        n
+    })
+}
+
+/// Gets fields_attribute from UserEntityDetail
+#[allow(dead_code)]
+pub fn get_attribute_from_url(
+    user_entity_detail: &Option<UserEntityDetail>,
+) -> Option<Vec<CredentialsFieldAttribute>> {
+    user_entity_detail.as_ref().and_then(|u| {
+        u.urls.first().and_then(|v| {
+            v.expanded_url.as_ref().map(|e| {
+                vec![CredentialsFieldAttribute {
+                    name: v.display_url.to_string(),
+                    value: e.to_string(),
+                }]
+            })
+        })
+    })
+}
+
 /// Generic register function
 /// As this function is supposed to be run only once, it will panic for every error it encounters
 /// Most of this function is a direct copy/paste of the official `elefren` crate
@@ -207,6 +241,63 @@ token = "{}""#,
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_get_attribute_from_url() {
+        let expected_credentials_field_attribute = CredentialsFieldAttribute {
+            name: "Nintendojo.fr".to_string(),
+            value: "https://www.nintendojo.fr".to_string(),
+        };
+
+        let true_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr".to_string()),
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        let false_urls = vec![UrlEntity {
+            display_url: "Nintendojo.fr".to_string(),
+            expanded_url: None,
+            range: (1, 3),
+            url: "https://t.me/balek".to_string(),
+        }];
+
+        assert!(get_attribute_from_url(&None).is_none());
+        assert!(get_attribute_from_url(&Some(UserEntityDetail { urls: false_urls })).is_none());
+
+        let binding = get_attribute_from_url(&Some(UserEntityDetail { urls: true_urls })).unwrap();
+        let result_credentials_field_attribute = binding.first().unwrap();
+
+        assert_eq!(
+            result_credentials_field_attribute.name,
+            expected_credentials_field_attribute.name
+        );
+        assert_eq!(
+            result_credentials_field_attribute.value,
+            expected_credentials_field_attribute.value
+        );
+    }
+
+    #[test]
+    fn test_get_note_from_description() {
+        let urls = vec![UrlEntity {
+            display_url: "tamerelol".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+            range: (1, 3),
+            url: "https://t.me/tamerelol".to_string(),
+        }];
+
+        let some_description = Some("Youpi | https://t.me/tamerelol".to_string());
+        let none_description = None;
+
+        assert_eq!(
+            get_note_from_description(&some_description, &urls),
+            Some("Youpi | https://www.nintendojo.fr/dojobar".to_string())
+        );
+
+        assert_eq!(get_note_from_description(&none_description, &urls), None);
+    }
+
     #[test]
     fn test_replace_tweet_by_toot() {
         let mut associated_urls = HashMap::from([

src/util.rs

@@ -1,5 +1,6 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
 
+use base64::encode;
 use egg_mode::tweet::Tweet;
 use futures::{stream, stream::StreamExt};
 use log::{error, info, warn};
@@ -71,6 +72,27 @@ pub async fn generate_media_ids(
     (media_url, media_ids)
 }
 
+/// Transforms the media into a base64 equivalent
+pub async fn base64_media(u: &str) -> Result<String, Box<dyn Error>> {
+    let mut response = reqwest::get(u).await?;
+
+    let mut buffer = Vec::new();
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut buffer).await?;
+    }
+
+    let content_type = response
+        .headers()
+        .get("content-type")
+        .ok_or_else(|| ScootalooError::new(&format!("Cannot get media content type for {}", u)))?
+        .to_str()?;
+
+    let encoded_f = encode(buffer);
+
+    Ok(format!("data:{};base64,{}", content_type, encoded_f))
+}
+
 /// Gets and caches Twitter Media inside the determined temp dir
 pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
     // create dir
@@ -129,4 +151,17 @@ mod tests {
 
         remove_dir_all(TMP_DIR).unwrap();
     }
+
+    #[tokio::test]
+    async fn test_base64_media() {
+        let img = base64_media(
+            "https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png",
+        )
+        .await
+        .unwrap();
+
+        assert!(img.starts_with("data:image/png;base64,"));
+
+        assert!(img.ends_with("="));
+    }
 }