From 5ce3bde3e73072e0640a4272b72e4dbc4ff017c1 Mon Sep 17 00:00:00 2001 From: VC Date: Mon, 7 Nov 2022 18:25:55 +0100 Subject: [PATCH 1/8] fix: remove unecessary \n in TOML conf --- src/mastodon.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mastodon.rs b/src/mastodon.rs index d903b0e..395f07b 100644 --- a/src/mastodon.rs +++ b/src/mastodon.rs @@ -101,9 +101,9 @@ pub fn register(host: &str, screen_name: &str) { println!( "Please insert the following block at the end of your configuration file: - \n[mastodon.{}] - \ntwitter_screen_name = \"{}\" - \n{}", + [mastodon.{}] + twitter_screen_name = \"{}\" + {}", screen_name.to_lowercase(), screen_name, toml From 62011b4b81caf2ad4c2a329101329071f3f910b0 Mon Sep 17 00:00:00 2001 From: VC Date: Mon, 7 Nov 2022 21:24:37 +0100 Subject: [PATCH 2/8] refactor: downloads/uploads every media from a tweet async way --- src/util.rs | 88 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/src/util.rs b/src/util.rs index 8aa34a8..af25269 100644 --- a/src/util.rs +++ b/src/util.rs @@ -15,45 +15,73 @@ pub async fn generate_media_ids( cache_path: &str, mastodon: &Mastodon, ) -> (String, Vec) { + let mut media_url = "".to_string(); let mut media_ids: Vec = vec![]; - let mut media_url: String = "".to_string(); if let Some(m) = &tweet.extended_entities { - for media in &m.media { - // attribute the media url + // create tasks list + let mut tasks = vec![]; + + // size of media_ids vector, should be equal to the media vector + media_ids.resize(m.media.len(), String::new()); + + for (i, media) in m.media.iter().enumerate() { + // attribute media url media_url = media.url.clone(); - let local_tweet_media_path = match get_tweet_media(media, cache_path).await { - Ok(m) => m, - Err(e) => { - error!("Cannot get tweet media for {}: {}", &media.url, e); - continue; - } - }; - let mastodon_media_ids = match mastodon - .media(Cow::from(local_tweet_media_path.to_owned())) - { - Ok(m) => { - remove_file(&local_tweet_media_path).await.unwrap_or_else(|e| - warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)); - m.id - } - Err(e) => { - error!( - "Attachment {} cannot be uploaded to Mastodon Instance: {}", - &local_tweet_media_path, e - ); - // file is no longer useful, deleting - remove_file(&local_tweet_media_path).await.unwrap_or_else(|e| - warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)); - continue; - } - }; + // clone everything we need + let cache_path = String::from(cache_path); + let media = media.clone(); + let mastodon = mastodon.clone(); - media_ids.push(mastodon_media_ids); + let task = tokio::task::spawn(async move { + // get the tweet embedded media + let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await { + Ok(l) => l, + Err(e) => { + return Err(ScootalooError::new(&format!( + "Cannot get tweet media for {}: {}", + &media.url, e + ))) + } + }; + + // upload media to Mastodon + let mastodon_media = + match mastodon.media(Cow::from(local_tweet_media_path.to_owned())) { + Ok(l) => l, + Err(e) => { + return Err(ScootalooError::new(&format!( + "Attachment {} cannot be uploaded to Mastodon Instance: {}", + &local_tweet_media_path, e + ))) + } + }; + + // at this point, we can safely erase the original file + if let Err(e) = remove_file(&local_tweet_media_path).await { + ScootalooError::new(&format!("Attachment {} has been uploaded but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)); + } + + Ok((i, mastodon_media.id)) + }); + + tasks.push(task); + } + + for task in tasks { + match task.await { + // insert the media at the right place + Ok(Ok((i, v))) => media_ids[i] = v, + Ok(Err(e)) => warn!("{}", e), + Err(e) => error!("Something went wrong when joining the main thread: {}", e), + } } } + // in case some media_ids slot remained empty due to errors, remove them + media_ids.retain(|x| !x.is_empty()); + (media_url, media_ids) } From 3b0e7234affeff2b6690e8171d36cf9a8fadb64e Mon Sep 17 00:00:00 2001 From: VC Date: Tue, 8 Nov 2022 08:37:17 +0100 Subject: [PATCH 3/8] refactor: downloads/uploads every media from a tweet async way --- src/util.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/util.rs b/src/util.rs index af25269..b2549e6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,9 +1,15 @@ use crate::{twitter::get_tweet_media, ScootalooError}; -use egg_mode::tweet::Tweet; -use elefren::prelude::*; -use log::{error, warn}; -use reqwest::Url; + use std::{borrow::Cow, error::Error}; + +use egg_mode::tweet::Tweet; + +use elefren::prelude::*; + +use log::{error, info, warn}; + +use reqwest::Url; + use tokio::{ fs::{create_dir_all, remove_file, File}, io::copy, @@ -35,6 +41,10 @@ pub async fn generate_media_ids( let mastodon = mastodon.clone(); let task = tokio::task::spawn(async move { +<<<<<<< HEAD +======= + info!("Start treating {}", media.media_url_https); +>>>>>>> 32f9605 (tamerelol) // get the tweet embedded media let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await { Ok(l) => l, @@ -51,6 +61,7 @@ pub async fn generate_media_ids( match mastodon.media(Cow::from(local_tweet_media_path.to_owned())) { Ok(l) => l, Err(e) => { + remove_file(&local_tweet_media_path).await.unwrap(); return Err(ScootalooError::new(&format!( "Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e From c62f67c3b3b7278795a8e455726dd4b4d686d9b4 Mon Sep 17 00:00:00 2001 From: VC Date: Mon, 7 Nov 2022 21:25:53 +0100 Subject: [PATCH 4/8] refactor: simpler mtask var --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4c68bb7..808b54d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,13 +23,13 @@ use elefren::{prelude::*, status_builder::StatusBuilder}; use log::{debug, info}; use rusqlite::Connection; use std::sync::Arc; -use tokio::{spawn, sync::Mutex, task::JoinHandle}; +use tokio::{spawn, sync::Mutex}; /// This is where the magic happens #[tokio::main] pub async fn run(config: Config) { // create the task vector for handling multiple accounts - let mut mtask: Vec> = vec![]; + let mut mtask = vec![]; // open the SQLite connection let conn = Arc::new(Mutex::new( From 3c64df23bcec74ff98661d8ec139d8bcc18e430c Mon Sep 17 00:00:00 2001 From: VC Date: Tue, 8 Nov 2022 08:37:53 +0100 Subject: [PATCH 5/8] refactor: add info/debug --- src/lib.rs | 8 ++++---- src/util.rs | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 808b54d..b7edac0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ pub use state::{init_db, migrate_db}; use state::{read_state, write_state, TweetToToot}; use elefren::{prelude::*, status_builder::StatusBuilder}; -use log::{debug, info}; +use log::info; use rusqlite::Connection; use std::sync::Arc; use tokio::{spawn, sync::Mutex}; @@ -48,7 +48,7 @@ pub async fn run(config: Config) { let task_conn = conn.clone(); let task = spawn(async move { - debug!("Starting treating {}", &mastodon_config.twitter_screen_name); + info!("Starting treating {}", &mastodon_config.twitter_screen_name); // retrieve the last tweet ID for the username let lconn = task_conn.lock().await; @@ -81,7 +81,7 @@ pub async fn run(config: Config) { feed.reverse(); for tweet in &feed { - debug!("Treating Tweet {} inside feed", tweet.id); + info!("Treating Tweet {} inside feed", tweet.id); // initiate the toot_reply_id var let mut toot_reply_id: Option = None; // determine if the tweet is part of a thread (response to self) or a standard response @@ -113,7 +113,7 @@ pub async fn run(config: Config) { status_text = status_text.replace(&media_url, ""); - debug!("Building corresponding Mastodon status"); + info!("Building corresponding Mastodon status"); let mut status_builder = StatusBuilder::new(); diff --git a/src/util.rs b/src/util.rs index b2549e6..d113fa5 100644 --- a/src/util.rs +++ b/src/util.rs @@ -31,6 +31,8 @@ pub async fn generate_media_ids( // size of media_ids vector, should be equal to the media vector media_ids.resize(m.media.len(), String::new()); + info!("{} medias in tweet", m.media.len()); + for (i, media) in m.media.iter().enumerate() { // attribute media url media_url = media.url.clone(); @@ -41,10 +43,7 @@ pub async fn generate_media_ids( let mastodon = mastodon.clone(); let task = tokio::task::spawn(async move { -<<<<<<< HEAD -======= info!("Start treating {}", media.media_url_https); ->>>>>>> 32f9605 (tamerelol) // get the tweet embedded media let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await { Ok(l) => l, @@ -65,7 +64,7 @@ pub async fn generate_media_ids( return Err(ScootalooError::new(&format!( "Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e - ))) + ))); } }; @@ -88,6 +87,8 @@ pub async fn generate_media_ids( Err(e) => error!("Something went wrong when joining the main thread: {}", e), } } + } else { + info!("No media in tweet"); } // in case some media_ids slot remained empty due to errors, remove them From 06904434c870609af5a078b53204c32ff6464e8c Mon Sep 17 00:00:00 2001 From: VC Date: Tue, 8 Nov 2022 08:32:36 +0100 Subject: [PATCH 6/8] fix: indentation error when registering --- src/mastodon.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mastodon.rs b/src/mastodon.rs index 395f07b..5434f40 100644 --- a/src/mastodon.rs +++ b/src/mastodon.rs @@ -101,9 +101,9 @@ pub fn register(host: &str, screen_name: &str) { println!( "Please insert the following block at the end of your configuration file: - [mastodon.{}] - twitter_screen_name = \"{}\" - {}", +[mastodon.{}] +twitter_screen_name = \"{}\" +{}", screen_name.to_lowercase(), screen_name, toml From 89f1372f9fba11e6240d187372e5897982e4cce0 Mon Sep 17 00:00:00 2001 From: VC Date: Tue, 8 Nov 2022 08:36:25 +0100 Subject: [PATCH 7/8] bump: version v0.8.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c801a9b..4e1461f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2016,7 +2016,7 @@ dependencies = [ [[package]] name = "scootaloo" -version = "0.7.2" +version = "0.8.0" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index d866bd5..0920dd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scootaloo" -version = "0.7.2" +version = "0.8.0" authors = ["VC "] edition = "2021" From 4415c4ac1254e9081abfa3ce0988301c497cc71a Mon Sep 17 00:00:00 2001 From: VC Date: Tue, 8 Nov 2022 10:54:42 +0100 Subject: [PATCH 8/8] refactor: better logic flow for uploading/deleting media --- src/util.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/util.rs b/src/util.rs index d113fa5..e994e2e 100644 --- a/src/util.rs +++ b/src/util.rs @@ -56,22 +56,20 @@ pub async fn generate_media_ids( }; // upload media to Mastodon - let mastodon_media = - match mastodon.media(Cow::from(local_tweet_media_path.to_owned())) { - Ok(l) => l, - Err(e) => { - remove_file(&local_tweet_media_path).await.unwrap(); - return Err(ScootalooError::new(&format!( - "Attachment {} cannot be uploaded to Mastodon Instance: {}", - &local_tweet_media_path, e - ))); - } - }; - + let mastodon_media = mastodon.media(Cow::from(local_tweet_media_path.to_owned())); // at this point, we can safely erase the original file - if let Err(e) = remove_file(&local_tweet_media_path).await { - ScootalooError::new(&format!("Attachment {} has been uploaded but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)); - } + // it doesn’t matter if we can’t remove, cache_media fn is idempotent + remove_file(&local_tweet_media_path).await.ok(); + + let mastodon_media = match mastodon_media { + Ok(m) => m, + Err(e) => { + return Err(ScootalooError::new(&format!( + "Attachment {} cannot be uploaded to Mastodon Instance: {}", + &local_tweet_media_path, e + ))) + } + }; Ok((i, mastodon_media.id)) });