dojo/scootaloo
1
0
Fork 0
mirror of https://framagit.org/veretcle/scootaloo.git synced 2025-07-21 09:31:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: use futures instead of tokio for media upload

Browse Source
This commit is contained in:
VC
2022-11-14 22:12:02 +01:00
parent 89de1cf7a3
commit ce84c05581
4 changed files with 41 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
src/util.rs
View File

@@ -15,6 +15,8 @@ use tokio::{
io::copy,
};
use futures::{stream, stream::StreamExt};
/// Generate associative table between media ids and tweet extended entities
pub async fn generate_media_ids(
tweet: &Tweet,
@@ -25,45 +27,40 @@ pub async fn generate_media_ids(
let mut media_ids: Vec<String> = vec![];
if let Some(m) = &tweet.extended_entities {
// create tasks list
let mut tasks = vec![];
// size of media_ids vector, should be equal to the media vector
media_ids.resize(m.media.len(), String::new());
info!("{} medias in tweet", m.media.len());
for (i, media) in m.media.iter().enumerate() {
// attribute media url
media_url = media.url.clone();
let medias = m.media.clone();
// clone everything we need
let cache_path = String::from(cache_path);
let media = media.clone();
let mastodon = mastodon.clone();
let mut stream = stream::iter(medias)
.map(|media| {
// attribute media url
media_url = media.url.clone();
let task = tokio::task::spawn(async move {
info!("Start treating {}", media.media_url_https);
// get the tweet embedded media
let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
// clone everything we need
let cache_path = String::from(cache_path);
let mastodon = mastodon.clone();
// upload media to Mastodon
let mastodon_media =
mastodon.media(Cow::from(local_tweet_media_path.to_owned()))?;
// at this point, we can safely erase the original file
// it doesn’t matter if we can’t remove, cache_media fn is idempotent
remove_file(&local_tweet_media_path).await.ok();
tokio::task::spawn(async move {
info!("Start treating {}", media.media_url_https);
// get the tweet embedded media
let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
Ok::<(usize, String), ScootalooError>((i, mastodon_media.id))
});
// upload media to Mastodon
let mastodon_media =
mastodon.media(Cow::from(local_tweet_media_path.to_owned()))?;
// at this point, we can safely erase the original file
// it doesn’t matter if we can’t remove, cache_media fn is idempotent
remove_file(&local_tweet_media_path).await.ok();
tasks.push(task);
}
Ok::<String, ScootalooError>(mastodon_media.id)
})
})
.buffered(4); // there are max four medias per tweet and they need to be treated in
// order
for task in tasks {
match task.await {
// insert the media at the right place
Ok(Ok((i, v))) => media_ids[i] = v,
while let Some(result) = stream.next().await {
match result {
Ok(Ok(v)) => media_ids.push(v),
Ok(Err(e)) => warn!("Cannot treat media: {}", e),
Err(e) => error!("Something went wrong when joining the main thread: {}", e),
}