From 891f46ec2f3d35f9bcad292f4eedd79ac8453d9d Mon Sep 17 00:00:00 2001 From: VC Date: Fri, 24 Jan 2025 13:49:11 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8:=20improve=20bsky=20image=20upload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 31 +++++++++++++++++- Cargo.toml | 3 +- src/bsky.rs | 91 +++++++++++++++++++++++++++++++++-------------------- 3 files changed, 89 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92da5fc..6db0279 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -733,6 +733,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -740,6 +755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -748,6 +764,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -783,6 +810,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1781,13 +1809,14 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oolatoocs" -version = "4.1.1" +version = "4.1.2" dependencies = [ "atrium-api", "bsky-sdk", "chrono", "clap", "env_logger", + "futures", "html-escape", "image", "log", diff --git a/Cargo.toml b/Cargo.toml index 6aac0d2..7f9841f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oolatoocs" -version = "4.1.1" +version = "4.1.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -9,6 +9,7 @@ edition = "2021" chrono = "^0.4" clap = "^4" env_logger = "^0.10" +futures = "^0.3" html-escape = "^0.2" log = "^0.4" megalodon = "^0.13" diff --git a/src/bsky.rs b/src/bsky.rs index afdcdbf..b711cb5 100644 --- a/src/bsky.rs +++ b/src/bsky.rs @@ -8,8 +8,9 @@ use bsky_sdk::{ rich_text::RichText, BskyAgent, }; +use futures::{stream, StreamExt}; use image::ImageReader; -use log::{debug, error}; +use log::{debug, error, warn}; use megalodon::entities::attachment::{Attachment, AttachmentType}; use regex::Regex; use std::{error::Error, fs::exists, io::Cursor}; @@ -152,39 +153,66 @@ pub async fn generate_media_records( let mut embed: Option< atrium_api::types::Union, > = None; - let mut images = Vec::new(); - let mut videos: Vec = Vec::new(); - for media in media_attach.iter() { - match media.r#type { - AttachmentType::Image => { - let blob = upload_media(true, bsky, &media.url).await.unwrap(); + let image_media_attach: Vec<_> = media_attach + .iter() + .filter(|x| x.r#type == AttachmentType::Image) + .cloned() + .collect(); + let video_media_attach: Vec<_> = media_attach + .iter() + .filter(|x| (x.r#type == AttachmentType::Video || x.r#type == AttachmentType::Gifv)) + .cloned() + .collect(); - images.push( + // Bsky only tasks 1 video per post, so we’ll try to treat that first and exit + if !video_media_attach.is_empty() { + // treat only the very first video, ignore the rest + let media = &video_media_attach[0]; + let blob = upload_media(false, bsky, &media.url).await.unwrap(); + + embed = Some(atrium_api::types::Union::Refs( + atrium_api::app::bsky::feed::post::RecordEmbedRefs::AppBskyEmbedVideoMain(Box::new( + atrium_api::app::bsky::embed::video::MainData { + alt: media.description.clone(), + aspect_ratio: None, + captions: None, + video: blob.data.blob, + } + .into(), + )), + )); + + // returns immediately, we don’t want to treat the other medias + return embed; + } + + let mut stream = stream::iter(image_media_attach) + .map(|media| { + let bsky = bsky.clone(); + tokio::task::spawn(async move { + debug!("Treating media {}", &media.url); + upload_media(true, &bsky, &media.url).await.map(|i| { atrium_api::app::bsky::embed::images::ImageData { alt: media .description .clone() .map_or("".to_string(), |v| v.to_owned()), aspect_ratio: None, - image: blob.data.blob, + image: i.data.blob, } - .into(), - ); - } - AttachmentType::Gifv | AttachmentType::Video => { - let blob = upload_media(false, bsky, &media.url).await.unwrap(); + }) + }) + }) + .buffered(4); - videos.push(atrium_api::app::bsky::embed::video::MainData { - alt: media.description.clone(), - aspect_ratio: None, - captions: None, - video: blob.data.blob, - }); - } - _ => { - error!("Not an image, not a video, what happened here?"); - } + let mut images = Vec::new(); + + while let Some(result) = stream.next().await { + match result { + Ok(Ok(v)) => images.push(v.into()), + Ok(Err(e)) => warn!("Cannot treat a specific media: {}", e), + Err(e) => error!("Something went wrong when joining main thread: {}", e), } } @@ -196,19 +224,14 @@ pub async fn generate_media_records( )); } - // if a video has been uploaded, it takes priority as you can only have 1 video per post - if !videos.is_empty() { - embed = Some(atrium_api::types::Union::Refs( - atrium_api::app::bsky::feed::post::RecordEmbedRefs::AppBskyEmbedVideoMain(Box::new( - videos[0].clone().into(), - )), - )) - } - embed } -async fn upload_media(is_image: bool, bsky: &BskyAgent, u: &str) -> Result> { +async fn upload_media( + is_image: bool, + bsky: &BskyAgent, + u: &str, +) -> Result> { let dl = reqwest::get(u).await?; let content_length = dl.content_length().ok_or("Content length unavailable")?; let bytes = if content_length <= 1_000_000 || !is_image {