Merge branch 'reply_improvements' into 'master'

refactor: improve reply/thread management

See merge request veretcle/scootaloo!25

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scootaloo"
-version = "0.7.2"
+version = "0.8.1"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 

src/error.rs

@@ -1,8 +1,12 @@
 use std::{
+    boxed::Box,
+    convert::From,
     error::Error,
     fmt::{Display, Formatter, Result},
 };
 
+use elefren::Error as elefrenError;
+
 #[derive(Debug)]
 pub struct ScootalooError {
     details: String,
@@ -23,3 +27,15 @@ impl Display for ScootalooError {
         write!(f, "{}", self.details)
     }
 }
+
+impl From<Box<dyn Error>> for ScootalooError {
+    fn from(error: Box<dyn Error>) -> Self {
+        ScootalooError::new(&format!("Error in a subset crate: {}", error))
+    }
+}
+
+impl From<elefrenError> for ScootalooError {
+    fn from(error: elefrenError) -> Self {
+        ScootalooError::new(&format!("Error in elefren crate: {}", error))
+    }
+}

src/lib.rs

@@ -20,16 +20,16 @@ pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
 use elefren::{prelude::*, status_builder::StatusBuilder};
-use log::{debug, info};
+use log::info;
 use rusqlite::Connection;
 use std::sync::Arc;
-use tokio::{spawn, sync::Mutex, task::JoinHandle};
+use tokio::{spawn, sync::Mutex};
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
     // create the task vector for handling multiple accounts
-    let mut mtask: Vec<JoinHandle<()>> = vec![];
+    let mut mtask = vec![];
 
     // open the SQLite connection
     let conn = Arc::new(Mutex::new(
@@ -48,40 +48,33 @@ pub async fn run(config: Config) {
         let task_conn = conn.clone();
 
         let task = spawn(async move {
-            debug!("Starting treating {}", &mastodon_config.twitter_screen_name);
+            info!("Starting treating {}", &mastodon_config.twitter_screen_name);
 
             // retrieve the last tweet ID for the username
             let lconn = task_conn.lock().await;
-            let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)
-                .unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
-                .map(|s| s.tweet_id);
+            let last_tweet_id =
+                read_state(&lconn, &mastodon_config.twitter_screen_name, None)?.map(|r| r.tweet_id);
             drop(lconn);
 
-            // get Mastodon instance
-            let mastodon = get_mastodon_token(&mastodon_config);
-
             // get user timeline feed (Vec<tweet>)
             let mut feed =
                 get_user_timeline(&mastodon_config.twitter_screen_name, &token, last_tweet_id)
-                    .await
-                    .unwrap_or_else(|e| {
-                        panic!(
-                            "Something went wrong when trying to retrieve {}’s timeline: {}",
-                            &mastodon_config.twitter_screen_name, e
-                        )
-                    });
+                    .await?;
 
             // empty feed -> exiting
             if feed.is_empty() {
                 info!("Nothing to retrieve since last time, exiting…");
-                return;
+                return Ok(());
             }
 
+            // get Mastodon instance
+            let mastodon = get_mastodon_token(&mastodon_config);
+
             // order needs to be chronological
             feed.reverse();
 
             for tweet in &feed {
-                debug!("Treating Tweet {} inside feed", tweet.id);
+                info!("Treating Tweet {} inside feed", tweet.id);
                 // initiate the toot_reply_id var
                 let mut toot_reply_id: Option<String> = None;
                 // determine if the tweet is part of a thread (response to self) or a standard response
@@ -113,7 +106,7 @@ pub async fn run(config: Config) {
 
                 status_text = status_text.replace(&media_url, "");
 
-                debug!("Building corresponding Mastodon status");
+                info!("Building corresponding Mastodon status");
 
                 let mut status_builder = StatusBuilder::new();
 
@@ -126,15 +119,11 @@ pub async fn run(config: Config) {
                 // can be activated for test purposes
                 // status_builder.visibility(elefren::status_builder::Visibility::Private);
 
-                let status = status_builder
-                    .build()
-                    .unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
+                let status = status_builder.build()?;
 
-                // publish status
-                // again unwrap is safe here as we are in the main thread
-                let published_status = mastodon.new_status(status).unwrap();
-                // this will panic if it cannot publish the status, which is a good thing, it allows the
-                // last_tweet gathered not to be written
+                let published_status = mastodon.new_status(status)?;
+                // this will return if it cannot publish the status preventing the last_tweet from
+                // being written into db
 
                 let ttt_towrite = TweetToToot {
                     twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
@@ -144,10 +133,10 @@ pub async fn run(config: Config) {
 
                 // write the current state (tweet ID and toot ID) to avoid copying it another time
                 let lconn = task_conn.lock().await;
-                write_state(&lconn, ttt_towrite)
-                    .unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e));
+                write_state(&lconn, ttt_towrite)?;
                 drop(lconn);
             }
+            Ok::<(), ScootalooError>(())
         });
 
         // push each task into the vec task
@@ -156,6 +145,10 @@ pub async fn run(config: Config) {
 
     // launch and wait for every handle
     for handle in mtask {
-        handle.await.unwrap();
+        match handle.await {
+            Ok(Err(e)) => eprintln!("Error within thread: {}", e),
+            Err(e) => eprintln!("Error with thread: {}", e),
+            _ => (),
+        }
     }
 }

src/mastodon.rs

@@ -101,9 +101,9 @@ pub fn register(host: &str, screen_name: &str) {
 
     println!(
         "Please insert the following block at the end of your configuration file:
-        \n[mastodon.{}]
-        \ntwitter_screen_name = \"{}\"
-        \n{}",
+[mastodon.{}]
+twitter_screen_name = \"{}\"
+{}",
         screen_name.to_lowercase(),
         screen_name,
         toml

src/util.rs

@@ -1,9 +1,15 @@
 use crate::{twitter::get_tweet_media, ScootalooError};
-use egg_mode::tweet::Tweet;
-use elefren::prelude::*;
-use log::{error, warn};
-use reqwest::Url;
+
 use std::{borrow::Cow, error::Error};
+
+use egg_mode::tweet::Tweet;
+
+use elefren::prelude::*;
+
+use log::{error, info, warn};
+
+use reqwest::Url;
+
 use tokio::{
     fs::{create_dir_all, remove_file, File},
     io::copy,
@@ -15,44 +21,76 @@ pub async fn generate_media_ids(
     cache_path: &str,
     mastodon: &Mastodon,
 ) -> (String, Vec<String>) {
+    let mut media_url = "".to_string();
     let mut media_ids: Vec<String> = vec![];
-    let mut media_url: String = "".to_string();
 
     if let Some(m) = &tweet.extended_entities {
-        for media in &m.media {
-            // attribute the media url
+        // create tasks list
+        let mut tasks = vec![];
+
+        // size of media_ids vector, should be equal to the media vector
+        media_ids.resize(m.media.len(), String::new());
+
+        info!("{} medias in tweet", m.media.len());
+
+        for (i, media) in m.media.iter().enumerate() {
+            // attribute media url
             media_url = media.url.clone();
-            let local_tweet_media_path = match get_tweet_media(media, cache_path).await {
+
+            // clone everything we need
+            let cache_path = String::from(cache_path);
+            let media = media.clone();
+            let mastodon = mastodon.clone();
+
+            let task = tokio::task::spawn(async move {
+                info!("Start treating {}", media.media_url_https);
+                // get the tweet embedded media
+                let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
+                    Ok(l) => l,
+                    Err(e) => {
+                        return Err(ScootalooError::new(&format!(
+                            "Cannot get tweet media for {}: {}",
+                            &media.url, e
+                        )))
+                    }
+                };
+
+                // upload media to Mastodon
+                let mastodon_media = mastodon.media(Cow::from(local_tweet_media_path.to_owned()));
+                // at this point, we can safely erase the original file
+                // it doesn’t matter if we can’t remove, cache_media fn is idempotent
+                remove_file(&local_tweet_media_path).await.ok();
+
+                let mastodon_media = match mastodon_media {
                     Ok(m) => m,
                     Err(e) => {
-                    error!("Cannot get tweet media for {}: {}", &media.url, e);
-                    continue;
-                }
-            };
-
-            let mastodon_media_ids = match mastodon
-                .media(Cow::from(local_tweet_media_path.to_owned()))
-            {
-                Ok(m) => {
-                    remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
-                        warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e));
-                    m.id
-                }
-                Err(e) => {
-                    error!(
+                        return Err(ScootalooError::new(&format!(
                             "Attachment {} cannot be uploaded to Mastodon Instance: {}",
                             &local_tweet_media_path, e
-                    );
-                    // file is no longer useful, deleting
-                    remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
-                        warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e));
-                    continue;
+                        )))
                     }
                 };
 
-            media_ids.push(mastodon_media_ids);
+                Ok((i, mastodon_media.id))
+            });
+
+            tasks.push(task);
+        }
+
+        for task in tasks {
+            match task.await {
+                // insert the media at the right place
+                Ok(Ok((i, v))) => media_ids[i] = v,
+                Ok(Err(e)) => warn!("{}", e),
+                Err(e) => error!("Something went wrong when joining the main thread: {}", e),
             }
         }
+    } else {
+        info!("No media in tweet");
+    }
+
+    // in case some media_ids slot remained empty due to errors, remove them
+    media_ids.retain(|x| !x.is_empty());
 
     (media_url, media_ids)
 }