Merge branch 'async_media_upload' into 'master'

Async media upload

See merge request veretcle/scootaloo!24

Cargo.lock

@@ -2016,7 +2016,7 @@ dependencies = [
 
 [[package]]
 name = "scootaloo"
-version = "0.7.2"
+version = "0.8.0"
 dependencies = [
  "chrono",
  "clap",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scootaloo"
-version = "0.7.2"
+version = "0.8.0"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 

src/lib.rs

@@ -20,16 +20,16 @@ pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
 use elefren::{prelude::*, status_builder::StatusBuilder};
-use log::{debug, info};
+use log::info;
 use rusqlite::Connection;
 use std::sync::Arc;
-use tokio::{spawn, sync::Mutex, task::JoinHandle};
+use tokio::{spawn, sync::Mutex};
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
     // create the task vector for handling multiple accounts
-    let mut mtask: Vec<JoinHandle<()>> = vec![];
+    let mut mtask = vec![];
 
     // open the SQLite connection
     let conn = Arc::new(Mutex::new(
@@ -48,7 +48,7 @@ pub async fn run(config: Config) {
         let task_conn = conn.clone();
 
         let task = spawn(async move {
-            debug!("Starting treating {}", &mastodon_config.twitter_screen_name);
+            info!("Starting treating {}", &mastodon_config.twitter_screen_name);
 
             // retrieve the last tweet ID for the username
             let lconn = task_conn.lock().await;
@@ -81,7 +81,7 @@ pub async fn run(config: Config) {
             feed.reverse();
 
             for tweet in &feed {
-                debug!("Treating Tweet {} inside feed", tweet.id);
+                info!("Treating Tweet {} inside feed", tweet.id);
                 // initiate the toot_reply_id var
                 let mut toot_reply_id: Option<String> = None;
                 // determine if the tweet is part of a thread (response to self) or a standard response
@@ -113,7 +113,7 @@ pub async fn run(config: Config) {
 
                 status_text = status_text.replace(&media_url, "");
 
-                debug!("Building corresponding Mastodon status");
+                info!("Building corresponding Mastodon status");
 
                 let mut status_builder = StatusBuilder::new();
 

src/mastodon.rs

@@ -101,9 +101,9 @@ pub fn register(host: &str, screen_name: &str) {
 
     println!(
         "Please insert the following block at the end of your configuration file:
-        \n[mastodon.{}]
+[mastodon.{}]
-        \ntwitter_screen_name = \"{}\"
+twitter_screen_name = \"{}\"
-        \n{}",
+{}",
         screen_name.to_lowercase(),
         screen_name,
         toml

src/util.rs

@@ -1,9 +1,15 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
-use egg_mode::tweet::Tweet;
+
-use elefren::prelude::*;
-use log::{error, warn};
-use reqwest::Url;
 use std::{borrow::Cow, error::Error};
+
+use egg_mode::tweet::Tweet;
+
+use elefren::prelude::*;
+
+use log::{error, info, warn};
+
+use reqwest::Url;
+
 use tokio::{
     fs::{create_dir_all, remove_file, File},
     io::copy,
@@ -15,45 +21,77 @@ pub async fn generate_media_ids(
     cache_path: &str,
     mastodon: &Mastodon,
 ) -> (String, Vec<String>) {
+    let mut media_url = "".to_string();
     let mut media_ids: Vec<String> = vec![];
-    let mut media_url: String = "".to_string();
 
     if let Some(m) = &tweet.extended_entities {
-        for media in &m.media {
+        // create tasks list
-            // attribute the media url
+        let mut tasks = vec![];
+
+        // size of media_ids vector, should be equal to the media vector
+        media_ids.resize(m.media.len(), String::new());
+
+        info!("{} medias in tweet", m.media.len());
+
+        for (i, media) in m.media.iter().enumerate() {
+            // attribute media url
             media_url = media.url.clone();
-            let local_tweet_media_path = match get_tweet_media(media, cache_path).await {
-                Ok(m) => m,
-                Err(e) => {
-                    error!("Cannot get tweet media for {}: {}", &media.url, e);
-                    continue;
-                }
-            };
 
-            let mastodon_media_ids = match mastodon
+            // clone everything we need
-                .media(Cow::from(local_tweet_media_path.to_owned()))
+            let cache_path = String::from(cache_path);
-            {
+            let media = media.clone();
-                Ok(m) => {
+            let mastodon = mastodon.clone();
-                    remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
-                        warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e));
-                    m.id
-                }
-                Err(e) => {
-                    error!(
-                        "Attachment {} cannot be uploaded to Mastodon Instance: {}",
-                        &local_tweet_media_path, e
-                    );
-                    // file is no longer useful, deleting
-                    remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
-                        warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e));
-                    continue;
-                }
-            };
 
-            media_ids.push(mastodon_media_ids);
+            let task = tokio::task::spawn(async move {
+                info!("Start treating {}", media.media_url_https);
+                // get the tweet embedded media
+                let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
+                    Ok(l) => l,
+                    Err(e) => {
+                        return Err(ScootalooError::new(&format!(
+                            "Cannot get tweet media for {}: {}",
+                            &media.url, e
+                        )))
+                    }
+                };
+
+                // upload media to Mastodon
+                let mastodon_media = mastodon.media(Cow::from(local_tweet_media_path.to_owned()));
+                // at this point, we can safely erase the original file
+                // it doesn’t matter if we can’t remove, cache_media fn is idempotent
+                remove_file(&local_tweet_media_path).await.ok();
+
+                let mastodon_media = match mastodon_media {
+                    Ok(m) => m,
+                    Err(e) => {
+                        return Err(ScootalooError::new(&format!(
+                            "Attachment {} cannot be uploaded to Mastodon Instance: {}",
+                            &local_tweet_media_path, e
+                        )))
+                    }
+                };
+
+                Ok((i, mastodon_media.id))
+            });
+
+            tasks.push(task);
         }
+
+        for task in tasks {
+            match task.await {
+                // insert the media at the right place
+                Ok(Ok((i, v))) => media_ids[i] = v,
+                Ok(Err(e)) => warn!("{}", e),
+                Err(e) => error!("Something went wrong when joining the main thread: {}", e),
+            }
+        }
+    } else {
+        info!("No media in tweet");
     }
 
+    // in case some media_ids slot remained empty due to errors, remove them
+    media_ids.retain(|x| !x.is_empty());
+
     (media_url, media_ids)
 }