dojo/scootaloo
1
0
Fork 0
mirror of https://framagit.org/veretcle/scootaloo.git synced 2025-07-20 17:11:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'noasync' into 'master'

Browse Source 
getting async in a reasonnable way

See merge request veretcle/scootaloo!12
This commit is contained in:
VC
2021-04-25 05:32:09 +00:00
parent da808b0051 c301649d49
commit 533a40f2c2
3 changed files with 492 additions and 698 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1089
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

13
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "scootaloo"
version = "0.3.3"
version = "0.4.0"
authors = ["VC <veretcle+framagit@mateu.be>"]
edition = "2018"
@@ -10,17 +10,12 @@ edition = "2018"
serde = { version = "1.0", features = ["derive"] }
toml = "^0.5"
clap = "^2.33"
futures = "^0.3"
egg-mode = { git = "https://github.com/egg-mode-rs/egg-mode", rev = "6b81073eba9c3b123ca0e80bdb5ef61d1758f131" }
elefren = "^0.22"
tokio = { version = "1", features = ["full"]}
reqwest = "^0.11"
elefren = "^0.20"
htmlescape = "^0.3"
reqwest = "^0.11"
log = "^0.4"
simple_logger = "^1.11"
[profile.release]
opt-level = 's' # Optimize for size.
lto = true # Link Time Optimization (LTO)
codegen-units = 1 # Set this to 1 to allow for maximum size reduction optimizations:
panic = 'abort' # removes the need for this extra unwinding code.

88
src/lib.rs
View File

@@ -6,7 +6,6 @@ use std::{
fmt,
fs::{read_to_string, write},
error::Error,
sync::{Arc, Mutex},
};
// toml
@@ -39,7 +38,6 @@ use reqwest::Url;
use tokio::{
io::copy,
fs::{File, create_dir_all, remove_file},
sync::mpsc,
};
// htmlescape
@@ -133,10 +131,10 @@ async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Err
return cache_media(&variant.url, t).await;
}
}
return Err(Box::new(ScootalooError::new(format!("Media Type for {} is video but no mp4 file URL is available", &m.url).as_str())));
return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
},
None => {
return Err(Box::new(ScootalooError::new(format!("Media Type for {} is video but does not contain any video_info", &m.url).as_str())));
return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
},
}
},
@@ -185,7 +183,6 @@ fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
/*
* Generic private functions
*/
/// Gets and caches Twitter Media inside the determined temp dir
async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
// create dir
@@ -196,8 +193,8 @@ async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
// create local file
let url = Url::parse(u)?;
let dest_filename = url.path_segments().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?
.last().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?;
let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
.last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
let dest_filepath = format!("{}/{}", t, dest_filename);
@@ -337,7 +334,7 @@ pub async fn run(config: Config) {
let token = get_oauth2_token(&config);
// get Mastodon instance
let mastodon = Arc::new(Mutex::new(get_mastodon_token(&config.mastodon)));
let mastodon = get_mastodon_token(&config.mastodon);
// get user timeline feed (Vec<tweet>)
let mut feed = get_user_timeline(&config, token, last_tweet_id)
@@ -377,61 +374,34 @@ pub async fn run(config: Config) {
let mut status_medias: Vec<String> = vec![];
// reupload the attachments if any
// reupload the attachments if any
if let Some(m) = &tweet.extended_entities {
let (tx, mut rx) = mpsc::channel(4);
for media in &m.media {
// creating a new tx for this initial loop
let tx = tx.clone();
// creating a new mastodon from the original mutex
let mastodon = mastodon.clone();
// unfortunately for this to be thread safe, we need to clone a lot of structures
let media = media.clone();
let cache_path = config.scootaloo.cache_path.clone();
let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path).await {
Ok(m) => m,
Err(e) => {
error!("Cannot get tweet media for {}: {}", &media.url, e);
continue;
},
};
tokio::spawn(async move {
debug!("Spawing new async thread to treat {}", &media.id);
let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
Ok(m) => m,
Err(e) => {
// we could have panicked here, no issue, but I’m not confortable using
// that for now
warn!("Cannot get tweet media for {}: {}", &media.url, e);
return;
}
};
// we cannot directly do all the stuff inside here because mastodon lock can
// live outside this
let mas_result = mastodon.lock().unwrap().media(Cow::from(String::from(&local_tweet_media_path)));
match mas_result {
Ok(m) => {
remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
warn!("Attachment {} has been uploaded but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
);
// we can unwrap here because we’re in a thread
tx.send(ScootalooSpawnResponse {
mastodon_media_id: m.id.clone(),
twitter_media_url: local_tweet_media_path.clone()
}).await.unwrap();
},
Err(e) => {
error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
}
let mastodon_media_ids = match mastodon.media(Cow::from(String::from(&local_tweet_media_path))) {
Ok(m) => {
remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
);
m.id
},
Err(e) => {
error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
continue;
}
});
}
};
// dropping the last tx otherwise recv() will wait indefinitely
drop(tx);
status_medias.push(mastodon_media_ids);
while let Some(i) = rx.recv().await {
// pushes the media into the media vec
status_medias.push(i.mastodon_media_id);
// removes the URL from the original Tweet text
status_text = status_text.replace(&i.twitter_media_url, "");
// last step, removing the reference to the media from with the toot’s text
status_text = status_text.replace(&media.url, "");
}
}
// finished reuploading attachments, now let’s do the toot baby!
@@ -441,11 +411,11 @@ pub async fn run(config: Config) {
.status(&status_text)
.media_ids(status_medias)
.build()
.expect(format!("Cannot build status with text {}", &status_text).as_str());
.expect(&format!("Cannot build status with text {}", &status_text));
// publish status
// again unwrap is safe here as we are in the main thread
mastodon.lock().unwrap().new_status(status).unwrap();
mastodon.new_status(status).unwrap();
// this will panic if it cannot publish the status, which is a good thing, it allows the
// last_tweet gathered not to be written