From eba13ba09532abd099112aefecd082cf92d02c9e Mon Sep 17 00:00:00 2001 From: VC Date: Thu, 9 Nov 2023 18:40:28 +0100 Subject: [PATCH 1/4] feat: add video/gif medias --- Cargo.lock | 2 +- Cargo.toml | 6 +- src/error.rs | 25 ++++ src/lib.rs | 47 ++++++- src/twitter.rs | 348 +++++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 411 insertions(+), 17 deletions(-) create mode 100644 src/error.rs diff --git a/Cargo.lock b/Cargo.lock index e6decbc..6314a48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1058,7 +1058,7 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oolatoocs" -version = "0.1.0" +version = "0.3.0" dependencies = [ "clap", "dissolve", diff --git a/Cargo.toml b/Cargo.toml index 71e8782..6d55b25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oolatoocs" -version = "0.1.0" +version = "0.3.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -13,10 +13,10 @@ log = "^0.4" megalodon = "^0.11" oauth1-request = "^0.6" regex = "1.10.2" -reqwest = { version = "0.11.22", features = ["json"] } +reqwest = { version = "0.11.22", features = ["json", "stream", "multipart"] } rusqlite = "^0.27" serde = { version = "^1.0", features = ["derive"] } -tokio = { version = "^1.33", features = ["rt-multi-thread", "macros"] } +tokio = { version = "^1.33", features = ["rt-multi-thread", "macros", "time"] } toml = "^0.8" [profile.release] diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..db7c95a --- /dev/null +++ b/src/error.rs @@ -0,0 +1,25 @@ +use std::{ + error::Error, + fmt::{Display, Formatter, Result}, +}; + +#[derive(Debug)] +pub struct OolatoocsError { + details: String, +} + +impl OolatoocsError { + pub fn new(msg: &str) -> OolatoocsError { + OolatoocsError { + details: msg.to_string(), + } + } +} + +impl Error for OolatoocsError {} + +impl Display for OolatoocsError { + fn fmt(&self, f: &mut Formatter) -> Result { + write!(f, "{}", self.details) + } +} diff --git a/src/lib.rs b/src/lib.rs index efc4c63..54de55a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ +mod error; +pub use error::OolatoocsError; + mod config; pub use config::{parse_toml, Config}; @@ -15,8 +18,9 @@ use utils::strip_everything; mod twitter; #[allow(unused_imports)] -use twitter::{post_tweet, upload_media}; +use twitter::{post_tweet, upload_chunk_media, upload_simple_media}; +use megalodon::entities::attachment::AttachmentType; use rusqlite::Connection; #[tokio::main] @@ -36,9 +40,46 @@ pub async fn run(config: &Config) { let Ok(tweet_content) = strip_everything(&toot.content, &toot.tags) else { continue; // skip in case we can’t strip something }; + let mut medias: Vec = vec![]; + // if we wanted to cut toot in half, now would be the right time to do so - // treating medias (nothing for now) - let tweet_id = post_tweet(&config.twitter, &tweet_content, &[]) + + for media in toot.media_attachments { + let id = match media.r#type { + AttachmentType::Image => { + let Ok(id) = + upload_simple_media(&config.twitter, &media.url, &media.description).await + else { + continue; + }; + id + } + AttachmentType::Gifv => { + let Ok(id) = upload_chunk_media(&config.twitter, &media.url, "tweet_gif").await + else { + continue; + }; + id + } + AttachmentType::Video => { + let Ok(id) = + upload_chunk_media(&config.twitter, &media.url, "tweet_video").await + else { + continue; + }; + id + } + _ => { + continue; + } + }; + + medias.push(id); + } + + println!("{:?}", medias); + + let tweet_id = post_tweet(&config.twitter, &tweet_content, &medias) .await .unwrap_or_else(|e| panic!("Cannot Tweet {}: {}", toot.id, e)); diff --git a/src/twitter.rs b/src/twitter.rs index 283d2d7..0e60136 100644 --- a/src/twitter.rs +++ b/src/twitter.rs @@ -1,16 +1,33 @@ use crate::config::TwitterConfig; +use crate::error::OolatoocsError; +use log::debug; use oauth1_request::Token; -use reqwest::Client; +use reqwest::{ + multipart::{Form, Part}, + Body, Client, +}; use serde::{Deserialize, Serialize}; use std::error::Error; +use tokio::time::{sleep, Duration}; -/// I don’t know, don’t ask me +const TWITTER_API_TWEET_URL: &str = "https://api.twitter.com/2/tweets"; +const TWITTER_UPLOAD_MEDIA_URL: &str = "https://upload.twitter.com/1.1/media/upload.json"; +const TWITTER_METADATA_MEDIA_URL: &str = + "https://upload.twitter.com/1.1/media/metadata/create.json"; + +// I don’t know, don’t ask me #[derive(oauth1_request::Request)] struct EmptyRequest {} #[derive(Serialize, Debug)] pub struct Tweet { pub text: String, + pub media: TweetMediasIds, +} + +#[derive(Serialize, Debug)] +pub struct TweetMediasIds { + pub media_ids: Vec, } #[derive(Deserialize, Debug)] @@ -23,6 +40,47 @@ pub struct TweetResponseData { pub id: String, } +#[derive(Deserialize, Debug)] +struct UploadMediaResponse { + media_id: u64, + processing_info: Option, +} + +#[derive(Deserialize, Debug)] +struct UploadMediaResponseProcessingInfo { + state: UploadMediaResponseProcessingInfoState, + check_after_secs: Option, +} + +#[derive(Deserialize, Debug)] +enum UploadMediaResponseProcessingInfoState { + #[serde(rename = "failed")] + Failed, + #[serde(rename = "succeeded")] + Succeeded, + #[serde(rename = "pending")] + Pending, + #[serde(rename = "in_progress")] + InProgress, +} + +#[derive(Serialize, Debug)] +struct MediaMetadata { + media_id: u64, + alt_text: MediaMetadataAltText, +} + +#[derive(Serialize, Debug)] +struct MediaMetadataAltText { + text: String, +} + +#[derive(Serialize, Debug, oauth1_request::Request)] +struct UploadMediaCommand { + command: String, + media_id: String, +} + /// This function returns the OAuth1 Token object from TwitterConfig fn get_token(config: &TwitterConfig) -> Token { oauth1_request::Token::from_parts( @@ -33,32 +91,302 @@ fn get_token(config: &TwitterConfig) -> Token { ) } -/// This function uploads media from Mastodon to Twitter and returns the media id from Twitter -#[allow(dead_code)] -pub async fn upload_media(_u: &str) -> Result> { - Ok(0) +/// This function uploads simple images from Mastodon to Twitter and returns the media id from Twitter +pub async fn upload_simple_media( + config: &TwitterConfig, + u: &str, + d: &Option, +) -> Result> { + // initiate request parameters + let empty_request = EmptyRequest {}; // Why? Because fuck you, that’s why! + let token = get_token(config); + + // retrieve the length and bytes stream from the given URL + let dl = reqwest::get(u).await?; + let content_length = dl + .content_length() + .ok_or(format!("Cannot get content length for {}", u))?; + let stream = dl.bytes_stream(); + + debug!("Ref download URL: {}", u); + + // upload the media + let client = Client::new(); + let res: UploadMediaResponse = client + .post(TWITTER_UPLOAD_MEDIA_URL) + .header( + "Authorization", + oauth1_request::post( + TWITTER_UPLOAD_MEDIA_URL, + &empty_request, + &token, + oauth1_request::HMAC_SHA1, + ), + ) + .multipart(Form::new().part( + "media", + Part::stream_with_length(Body::wrap_stream(stream), content_length), + )) + .send() + .await? + .json() + .await?; + + debug!("Media ID: {}", res.media_id); + + // update the metadata + if let Some(metadata) = d { + debug!("Metadata found! Processing…"); + metadata_create(config, res.media_id, metadata).await?; + } + + Ok(res.media_id) +} + +/// This function updates the metadata given the current media_id and token +async fn metadata_create(config: &TwitterConfig, id: u64, m: &str) -> Result<(), Box> { + let token = get_token(config); + let empty_request = EmptyRequest {}; + + let media_metadata = MediaMetadata { + media_id: id, + alt_text: MediaMetadataAltText { + text: m.to_string(), + }, + }; + + debug!("Metadata to process: {}", m); + + let client = Client::new(); + let metadata = client + .post(TWITTER_METADATA_MEDIA_URL) + .header( + "Authorization", + oauth1_request::post( + TWITTER_METADATA_MEDIA_URL, + &empty_request, + &token, + oauth1_request::HMAC_SHA1, + ), + ) + .json(&media_metadata) + .send() + .await?; + + debug!("Metadata processed with return code: {}", metadata.status()); + + Ok(()) +} + +/// This posts video/gif to Twitter and returns the media id from Twitter +pub async fn upload_chunk_media( + config: &TwitterConfig, + u: &str, + t: &str, +) -> Result> { + let empty_request = EmptyRequest {}; + let token = get_token(config); + + // retrieve the length, type and bytes stream from the given URL + let mut dl = reqwest::get(u).await?; + let content_length = dl + .content_length() + .ok_or(format!("Cannot get content length for {}", u))?; + let content_headers = dl.headers().clone(); + let content_type = content_headers + .get("Content-Type") + .ok_or(format!("Cannot get content type for {}", u))? + .to_str()?; + + debug!("Init the slot for uploading media: {}", u); + // init the slot for uploading + let client = Client::new(); + let orig_media_id: UploadMediaResponse = client + .post(TWITTER_UPLOAD_MEDIA_URL) + .header( + "Authorization", + oauth1_request::post( + TWITTER_UPLOAD_MEDIA_URL, + &empty_request, + &token, + oauth1_request::HMAC_SHA1, + ), + ) + .multipart( + Form::new() + .text("command", "INIT") + .text("media_type", content_type.to_owned()) + .text("total_bytes", content_length.to_string()) + .text("media_category", t.to_string()), + ) + .send() + .await? + .json() + .await?; + + debug!("Slot initiated with ID: {}", orig_media_id.media_id); + + debug!("Appending media to ID: {}", orig_media_id.media_id); + // append the media to the corresponding slot + let mut segment: u8 = 0; + while let Some(chunk) = dl.chunk().await? { + debug!( + "Appending segment {} for media ID {}", + segment, orig_media_id.media_id + ); + let chunk_size: u64 = chunk.len().try_into().unwrap(); + let res = client + .post(TWITTER_UPLOAD_MEDIA_URL) + .header( + "Authorization", + oauth1_request::post( + TWITTER_UPLOAD_MEDIA_URL, + &empty_request, + &token, + oauth1_request::HMAC_SHA1, + ), + ) + .multipart( + Form::new() + .text("command", "APPEND") + .text("media_id", orig_media_id.media_id.to_string()) + .text("segment_index", segment.to_string()) + .part("media", Part::stream_with_length(chunk, chunk_size)), + ) + .send() + .await?; + + if !res.status().is_success() { + return Err( + OolatoocsError::new(&format!("Cannot upload part {} of {}", segment, u)).into(), + ); + } + + segment += 1; + } + + debug!("Finalize media ID: {}", orig_media_id.media_id); + // Finalizing task + let fin: UploadMediaResponse = client + .post(TWITTER_UPLOAD_MEDIA_URL) + .header( + "Authorization", + oauth1_request::post( + TWITTER_UPLOAD_MEDIA_URL, + &empty_request, + &token, + oauth1_request::HMAC_SHA1, + ), + ) + .multipart( + Form::new() + .text("command", "FINALIZE") + .text("media_id", orig_media_id.media_id.to_string()), + ) + .send() + .await? + .json() + .await?; + + if let Some(p_info) = fin.processing_info { + if let Some(wait_sec) = p_info.check_after_secs { + debug!( + "Processing is not finished yet for ID {}, waiting {} secs", + orig_media_id.media_id, wait_sec + ); + // getting here, we have a status and a check_after_secs + // this status can be anything but we will check it afterwards + // whatever happens, we can wait here before proceeding + sleep(Duration::from_secs(wait_sec)).await; + + let command = UploadMediaCommand { + command: "STATUS".to_string(), + media_id: orig_media_id.media_id.to_string(), + }; + + loop { + debug!( + "Checking on status for ID {} after waiting {} secs", + orig_media_id.media_id, wait_sec + ); + + let status: UploadMediaResponse = client + .get(TWITTER_UPLOAD_MEDIA_URL) + .header( + "Authorization", + oauth1_request::get( + TWITTER_UPLOAD_MEDIA_URL, + &command, + &token, + oauth1_request::HMAC_SHA1, + ), + ) + .query(&command) + .send() + .await? + .json() + .await?; + + let p_status = status.processing_info.unwrap(); // shouldn’t be None at this point + match p_status.state { + UploadMediaResponseProcessingInfoState::Failed => { + debug!("Processing has failed!"); + return Err(OolatoocsError::new(&format!( + "Upload for {} (id: {}) has failed", + u, orig_media_id.media_id + )) + .into()); + } + UploadMediaResponseProcessingInfoState::Succeeded => { + debug!("Processing has succeeded, exiting loop!"); + break; + } + UploadMediaResponseProcessingInfoState::Pending + | UploadMediaResponseProcessingInfoState::InProgress => { + debug!( + "Processing still pending, waiting {} secs more…", + p_status.check_after_secs.unwrap() // unwrap is safe here, + // check_after_secs is only present + // when status is pending + ); + sleep(Duration::from_secs(p_status.check_after_secs.unwrap())).await; + continue; + } + } + } + } + } + + Ok(orig_media_id.media_id) } /// This posts Tweets with all the associated medias pub async fn post_tweet( config: &TwitterConfig, content: &str, - _medias: &[u64], + medias: &[u64], ) -> Result> { - let uri = "https://api.twitter.com/2/tweets"; let empty_request = EmptyRequest {}; // Why? Because fuck you, that’s why! let token = get_token(config); let tweet = Tweet { text: content.to_string(), + media: TweetMediasIds { + media_ids: medias.iter().map(|m| m.to_string()).collect(), + }, }; let client = Client::new(); let res = client - .post(uri) + .post(TWITTER_API_TWEET_URL) .header( "Authorization", - oauth1_request::post(uri, &empty_request, &token, oauth1_request::HMAC_SHA1), + oauth1_request::post( + TWITTER_API_TWEET_URL, + &empty_request, + &token, + oauth1_request::HMAC_SHA1, + ), ) .json(&tweet) .send() From b9179d8cce311a609a7ee8fcfb529ccdfc0fe11f Mon Sep 17 00:00:00 2001 From: VC Date: Sat, 11 Nov 2023 10:25:41 +0100 Subject: [PATCH 2/4] feat: threads --- src/lib.rs | 13 ++++++++++-- src/twitter.rs | 55 +++++++++++++++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 54de55a..5829080 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,7 @@ pub async fn run(config: &Config) { // if we wanted to cut toot in half, now would be the right time to do so + // treats media for media in toot.media_attachments { let id = match media.r#type { AttachmentType::Image => { @@ -77,12 +78,20 @@ pub async fn run(config: &Config) { medias.push(id); } - println!("{:?}", medias); + // threads if necessary + let reply_to = toot.in_reply_to_id.and_then(|t| { + read_state(&conn, Some(t.parse::().unwrap())) + .ok() + .flatten() + .map(|s| s.tweet_id) + }); - let tweet_id = post_tweet(&config.twitter, &tweet_content, &medias) + // posts corresponding tweet + let tweet_id = post_tweet(&config.twitter, &tweet_content, &medias, &reply_to) .await .unwrap_or_else(|e| panic!("Cannot Tweet {}: {}", toot.id, e)); + // writes the current state of the tweet write_state( &conn, TweetToToot { diff --git a/src/twitter.rs b/src/twitter.rs index 0e60136..8078c90 100644 --- a/src/twitter.rs +++ b/src/twitter.rs @@ -7,7 +7,7 @@ use reqwest::{ Body, Client, }; use serde::{Deserialize, Serialize}; -use std::error::Error; +use std::{error::Error, ops::Not}; use tokio::time::{sleep, Duration}; const TWITTER_API_TWEET_URL: &str = "https://api.twitter.com/2/tweets"; @@ -20,24 +20,32 @@ const TWITTER_METADATA_MEDIA_URL: &str = struct EmptyRequest {} #[derive(Serialize, Debug)] -pub struct Tweet { - pub text: String, - pub media: TweetMediasIds, +struct Tweet { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + media: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reply: Option, } #[derive(Serialize, Debug)] -pub struct TweetMediasIds { - pub media_ids: Vec, +struct TweetMediasIds { + media_ids: Vec, +} + +#[derive(Serialize, Debug)] +struct TweetReply { + in_reply_to_tweet_id: String, } #[derive(Deserialize, Debug)] -pub struct TweetResponse { - pub data: TweetResponseData, +struct TweetResponse { + data: TweetResponseData, } #[derive(Deserialize, Debug)] -pub struct TweetResponseData { - pub id: String, +struct TweetResponseData { + id: String, } #[derive(Deserialize, Debug)] @@ -112,7 +120,7 @@ pub async fn upload_simple_media( // upload the media let client = Client::new(); - let res: UploadMediaResponse = client + let res = client .post(TWITTER_UPLOAD_MEDIA_URL) .header( "Authorization", @@ -129,7 +137,7 @@ pub async fn upload_simple_media( )) .send() .await? - .json() + .json::() .await?; debug!("Media ID: {}", res.media_id); @@ -201,7 +209,7 @@ pub async fn upload_chunk_media( debug!("Init the slot for uploading media: {}", u); // init the slot for uploading let client = Client::new(); - let orig_media_id: UploadMediaResponse = client + let orig_media_id = client .post(TWITTER_UPLOAD_MEDIA_URL) .header( "Authorization", @@ -221,7 +229,7 @@ pub async fn upload_chunk_media( ) .send() .await? - .json() + .json::() .await?; debug!("Slot initiated with ID: {}", orig_media_id.media_id); @@ -267,7 +275,7 @@ pub async fn upload_chunk_media( debug!("Finalize media ID: {}", orig_media_id.media_id); // Finalizing task - let fin: UploadMediaResponse = client + let fin = client .post(TWITTER_UPLOAD_MEDIA_URL) .header( "Authorization", @@ -285,7 +293,7 @@ pub async fn upload_chunk_media( ) .send() .await? - .json() + .json::() .await?; if let Some(p_info) = fin.processing_info { @@ -310,7 +318,7 @@ pub async fn upload_chunk_media( orig_media_id.media_id, wait_sec ); - let status: UploadMediaResponse = client + let status = client .get(TWITTER_UPLOAD_MEDIA_URL) .header( "Authorization", @@ -324,7 +332,7 @@ pub async fn upload_chunk_media( .query(&command) .send() .await? - .json() + .json::() .await?; let p_status = status.processing_info.unwrap(); // shouldn’t be None at this point @@ -347,7 +355,8 @@ pub async fn upload_chunk_media( "Processing still pending, waiting {} secs more…", p_status.check_after_secs.unwrap() // unwrap is safe here, // check_after_secs is only present - // when status is pending + // when status is pending or in + // progress ); sleep(Duration::from_secs(p_status.check_after_secs.unwrap())).await; continue; @@ -365,15 +374,19 @@ pub async fn post_tweet( config: &TwitterConfig, content: &str, medias: &[u64], + reply_to: &Option, ) -> Result> { let empty_request = EmptyRequest {}; // Why? Because fuck you, that’s why! let token = get_token(config); let tweet = Tweet { text: content.to_string(), - media: TweetMediasIds { + media: medias.is_empty().not().then(|| TweetMediasIds { media_ids: medias.iter().map(|m| m.to_string()).collect(), - }, + }), + reply: reply_to.map(|s| TweetReply { + in_reply_to_tweet_id: s.to_string(), + }), }; let client = Client::new(); From af7156786b80c59ba4106232b20e199f24858b33 Mon Sep 17 00:00:00 2001 From: VC Date: Sat, 11 Nov 2023 12:16:57 +0100 Subject: [PATCH 3/4] chore: bump version to v1.0.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6314a48..48a5868 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1058,7 +1058,7 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oolatoocs" -version = "0.3.0" +version = "1.0.0" dependencies = [ "clap", "dissolve", diff --git a/Cargo.toml b/Cargo.toml index 6d55b25..7129e51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oolatoocs" -version = "0.3.0" +version = "1.0.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 6d208f3de37f7e17a6ea0e32c4b49590227254a4 Mon Sep 17 00:00:00 2001 From: VC Date: Sat, 11 Nov 2023 12:59:57 +0100 Subject: [PATCH 4/4] tamerelol --- src/twitter.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/twitter.rs b/src/twitter.rs index ec1bb95..8078c90 100644 --- a/src/twitter.rs +++ b/src/twitter.rs @@ -89,22 +89,6 @@ struct UploadMediaCommand { media_id: String, } -#[derive(Deserialize, Debug)] -struct UploadMediaResponse { - media_id: u64, -} - -#[derive(Serialize, Debug)] -struct MediaMetadata { - media_id: u64, - alt_text: MediaMetadataAltText, -} - -#[derive(Serialize, Debug)] -struct MediaMetadataAltText { - text: String, -} - /// This function returns the OAuth1 Token object from TwitterConfig fn get_token(config: &TwitterConfig) -> Token { oauth1_request::Token::from_parts(