Merge branch 'feat_other_medias' into 'main'

feat: separate metadata create + modify upload media for simple media only

See merge request veretcle/oolatoocs!2

Cargo.lock

@@ -1058,7 +1058,7 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "oolatoocs"
-version = "0.2.0"
+version = "1.0.0"
 dependencies = [
  "clap",
  "dissolve",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "oolatoocs"
-version = "0.2.0"
+version = "1.0.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -16,7 +16,7 @@ regex = "1.10.2"
 reqwest = { version = "0.11.22", features = ["json", "stream", "multipart"] }
 rusqlite = "^0.27"
 serde = { version = "^1.0", features = ["derive"] }
-tokio = { version = "^1.33", features = ["rt-multi-thread", "macros"] }
+tokio = { version = "^1.33", features = ["rt-multi-thread", "macros", "time"] }
 toml = "^0.8"
 
 [profile.release]

src/error.rs

@@ -0,0 +1,25 @@
+use std::{
+    error::Error,
+    fmt::{Display, Formatter, Result},
+};
+
+#[derive(Debug)]
+pub struct OolatoocsError {
+    details: String,
+}
+
+impl OolatoocsError {
+    pub fn new(msg: &str) -> OolatoocsError {
+        OolatoocsError {
+            details: msg.to_string(),
+        }
+    }
+}
+
+impl Error for OolatoocsError {}
+
+impl Display for OolatoocsError {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        write!(f, "{}", self.details)
+    }
+}

src/lib.rs

@@ -1,3 +1,6 @@
+mod error;
+pub use error::OolatoocsError;
+
 mod config;
 pub use config::{parse_toml, Config};
 
@@ -15,8 +18,9 @@ use utils::strip_everything;
 
 mod twitter;
 #[allow(unused_imports)]
-use twitter::{post_tweet, upload_media};
+use twitter::{post_tweet, upload_chunk_media, upload_simple_media};
 
+use megalodon::entities::attachment::AttachmentType;
 use rusqlite::Connection;
 
 #[tokio::main]
@@ -40,18 +44,54 @@ pub async fn run(config: &Config) {
 
         // if we wanted to cut toot in half, now would be the right time to do so
 
+        // treats media
         for media in toot.media_attachments {
-            let Ok(id) = upload_media(&config.twitter, &media.url, &media.description).await else {
+            let id = match media.r#type {
+                AttachmentType::Image => {
+                    let Ok(id) =
+                        upload_simple_media(&config.twitter, &media.url, &media.description).await
+                    else {
                         continue;
                     };
+                    id
+                }
+                AttachmentType::Gifv => {
+                    let Ok(id) = upload_chunk_media(&config.twitter, &media.url, "tweet_gif").await
+                    else {
+                        continue;
+                    };
+                    id
+                }
+                AttachmentType::Video => {
+                    let Ok(id) =
+                        upload_chunk_media(&config.twitter, &media.url, "tweet_video").await
+                    else {
+                        continue;
+                    };
+                    id
+                }
+                _ => {
+                    continue;
+                }
+            };
 
             medias.push(id);
         }
 
-        let tweet_id = post_tweet(&config.twitter, &tweet_content, &medias)
+        // threads if necessary
+        let reply_to = toot.in_reply_to_id.and_then(|t| {
+            read_state(&conn, Some(t.parse::<u64>().unwrap()))
+                .ok()
+                .flatten()
+                .map(|s| s.tweet_id)
+        });
+
+        // posts corresponding tweet
+        let tweet_id = post_tweet(&config.twitter, &tweet_content, &medias, &reply_to)
             .await
             .unwrap_or_else(|e| panic!("Cannot Tweet {}: {}", toot.id, e));
 
+        // writes the current state of the tweet
         write_state(
             &conn,
             TweetToToot {

src/twitter.rs

@@ -1,40 +1,75 @@
 use crate::config::TwitterConfig;
+use crate::error::OolatoocsError;
+use log::debug;
 use oauth1_request::Token;
 use reqwest::{
     multipart::{Form, Part},
     Body, Client,
 };
 use serde::{Deserialize, Serialize};
-use std::error::Error;
+use std::{error::Error, ops::Not};
+use tokio::time::{sleep, Duration};
 
-/// I don’t know, don’t ask me
+const TWITTER_API_TWEET_URL: &str = "https://api.twitter.com/2/tweets";
+const TWITTER_UPLOAD_MEDIA_URL: &str = "https://upload.twitter.com/1.1/media/upload.json";
+const TWITTER_METADATA_MEDIA_URL: &str =
+    "https://upload.twitter.com/1.1/media/metadata/create.json";
+
+// I don’t know, don’t ask me
 #[derive(oauth1_request::Request)]
 struct EmptyRequest {}
 
 #[derive(Serialize, Debug)]
-pub struct Tweet {
+struct Tweet {
-    pub text: String,
+    text: String,
-    pub media: TweetMediasIds,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    media: Option<TweetMediasIds>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    reply: Option<TweetReply>,
 }
 
 #[derive(Serialize, Debug)]
-pub struct TweetMediasIds {
+struct TweetMediasIds {
-    pub media_ids: Vec<String>,
+    media_ids: Vec<String>,
+}
+
+#[derive(Serialize, Debug)]
+struct TweetReply {
+    in_reply_to_tweet_id: String,
 }
 
 #[derive(Deserialize, Debug)]
-pub struct TweetResponse {
+struct TweetResponse {
-    pub data: TweetResponseData,
+    data: TweetResponseData,
 }
 
 #[derive(Deserialize, Debug)]
-pub struct TweetResponseData {
+struct TweetResponseData {
-    pub id: String,
+    id: String,
 }
 
 #[derive(Deserialize, Debug)]
 struct UploadMediaResponse {
     media_id: u64,
+    processing_info: Option<UploadMediaResponseProcessingInfo>,
+}
+
+#[derive(Deserialize, Debug)]
+struct UploadMediaResponseProcessingInfo {
+    state: UploadMediaResponseProcessingInfoState,
+    check_after_secs: Option<u64>,
+}
+
+#[derive(Deserialize, Debug)]
+enum UploadMediaResponseProcessingInfoState {
+    #[serde(rename = "failed")]
+    Failed,
+    #[serde(rename = "succeeded")]
+    Succeeded,
+    #[serde(rename = "pending")]
+    Pending,
+    #[serde(rename = "in_progress")]
+    InProgress,
 }
 
 #[derive(Serialize, Debug)]
@@ -48,6 +83,12 @@ struct MediaMetadataAltText {
     text: String,
 }
 
+#[derive(Serialize, Debug, oauth1_request::Request)]
+struct UploadMediaCommand {
+    command: String,
+    media_id: String,
+}
+
 /// This function returns the OAuth1 Token object from TwitterConfig
 fn get_token(config: &TwitterConfig) -> Token {
     oauth1_request::Token::from_parts(
@@ -58,32 +99,37 @@ fn get_token(config: &TwitterConfig) -> Token {
     )
 }
 
-/// This function uploads media from Mastodon to Twitter and returns the media id from Twitter
+/// This function uploads simple images from Mastodon to Twitter and returns the media id from Twitter
-#[allow(dead_code)]
+pub async fn upload_simple_media(
-pub async fn upload_media(
     config: &TwitterConfig,
     u: &str,
     d: &Option<String>,
 ) -> Result<u64, Box<dyn Error>> {
     // initiate request parameters
-    let uri = "https://upload.twitter.com/1.1/media/upload.json";
     let empty_request = EmptyRequest {}; // Why? Because fuck you, that’s why!
     let token = get_token(config);
 
-    // retrieve the length, type and bytes stream from the given URL
+    // retrieve the length and bytes stream from the given URL
     let dl = reqwest::get(u).await?;
     let content_length = dl
         .content_length()
         .ok_or(format!("Cannot get content length for {}", u))?;
     let stream = dl.bytes_stream();
 
+    debug!("Ref download URL: {}", u);
+
     // upload the media
     let client = Client::new();
-    let res: UploadMediaResponse = client
+    let res = client
-        .post(uri)
+        .post(TWITTER_UPLOAD_MEDIA_URL)
         .header(
             "Authorization",
-            oauth1_request::post(uri, &empty_request, &token, oauth1_request::HMAC_SHA1),
+            oauth1_request::post(
+                TWITTER_UPLOAD_MEDIA_URL,
+                &empty_request,
+                &token,
+                oauth1_request::HMAC_SHA1,
+            ),
         )
         .multipart(Form::new().part(
             "media",
@@ -91,30 +137,236 @@ pub async fn upload_media(
         ))
         .send()
         .await?
-        .json()
+        .json::<UploadMediaResponse>()
         .await?;
 
+    debug!("Media ID: {}", res.media_id);
+
     // update the metadata
     if let Some(metadata) = d {
-        let uri = "https://upload.twitter.com/1.1/media/metadata/create.json";
+        debug!("Metadata found! Processing…");
+        metadata_create(config, res.media_id, metadata).await?;
+    }
+
+    Ok(res.media_id)
+}
+
+/// This function updates the metadata given the current media_id and token
+async fn metadata_create(config: &TwitterConfig, id: u64, m: &str) -> Result<(), Box<dyn Error>> {
+    let token = get_token(config);
+    let empty_request = EmptyRequest {};
+
     let media_metadata = MediaMetadata {
-            media_id: res.media_id,
+        media_id: id,
         alt_text: MediaMetadataAltText {
-                text: metadata.to_string(),
+            text: m.to_string(),
         },
     };
-        let _metadata = client
+
-            .post(uri)
+    debug!("Metadata to process: {}", m);
+
+    let client = Client::new();
+    let metadata = client
+        .post(TWITTER_METADATA_MEDIA_URL)
         .header(
             "Authorization",
-                oauth1_request::post(uri, &empty_request, &token, oauth1_request::HMAC_SHA1),
+            oauth1_request::post(
+                TWITTER_METADATA_MEDIA_URL,
+                &empty_request,
+                &token,
+                oauth1_request::HMAC_SHA1,
+            ),
         )
         .json(&media_metadata)
         .send()
         .await?;
+
+    debug!("Metadata processed with return code: {}", metadata.status());
+
+    Ok(())
+}
+
+/// This posts video/gif to Twitter and returns the media id from Twitter
+pub async fn upload_chunk_media(
+    config: &TwitterConfig,
+    u: &str,
+    t: &str,
+) -> Result<u64, Box<dyn Error>> {
+    let empty_request = EmptyRequest {};
+    let token = get_token(config);
+
+    // retrieve the length, type and bytes stream from the given URL
+    let mut dl = reqwest::get(u).await?;
+    let content_length = dl
+        .content_length()
+        .ok_or(format!("Cannot get content length for {}", u))?;
+    let content_headers = dl.headers().clone();
+    let content_type = content_headers
+        .get("Content-Type")
+        .ok_or(format!("Cannot get content type for {}", u))?
+        .to_str()?;
+
+    debug!("Init the slot for uploading media: {}", u);
+    // init the slot for uploading
+    let client = Client::new();
+    let orig_media_id = client
+        .post(TWITTER_UPLOAD_MEDIA_URL)
+        .header(
+            "Authorization",
+            oauth1_request::post(
+                TWITTER_UPLOAD_MEDIA_URL,
+                &empty_request,
+                &token,
+                oauth1_request::HMAC_SHA1,
+            ),
+        )
+        .multipart(
+            Form::new()
+                .text("command", "INIT")
+                .text("media_type", content_type.to_owned())
+                .text("total_bytes", content_length.to_string())
+                .text("media_category", t.to_string()),
+        )
+        .send()
+        .await?
+        .json::<UploadMediaResponse>()
+        .await?;
+
+    debug!("Slot initiated with ID: {}", orig_media_id.media_id);
+
+    debug!("Appending media to ID: {}", orig_media_id.media_id);
+    // append the media to the corresponding slot
+    let mut segment: u8 = 0;
+    while let Some(chunk) = dl.chunk().await? {
+        debug!(
+            "Appending segment {} for media ID {}",
+            segment, orig_media_id.media_id
+        );
+        let chunk_size: u64 = chunk.len().try_into().unwrap();
+        let res = client
+            .post(TWITTER_UPLOAD_MEDIA_URL)
+            .header(
+                "Authorization",
+                oauth1_request::post(
+                    TWITTER_UPLOAD_MEDIA_URL,
+                    &empty_request,
+                    &token,
+                    oauth1_request::HMAC_SHA1,
+                ),
+            )
+            .multipart(
+                Form::new()
+                    .text("command", "APPEND")
+                    .text("media_id", orig_media_id.media_id.to_string())
+                    .text("segment_index", segment.to_string())
+                    .part("media", Part::stream_with_length(chunk, chunk_size)),
+            )
+            .send()
+            .await?;
+
+        if !res.status().is_success() {
+            return Err(
+                OolatoocsError::new(&format!("Cannot upload part {} of {}", segment, u)).into(),
+            );
         }
 
-    Ok(res.media_id)
+        segment += 1;
+    }
+
+    debug!("Finalize media ID: {}", orig_media_id.media_id);
+    // Finalizing task
+    let fin = client
+        .post(TWITTER_UPLOAD_MEDIA_URL)
+        .header(
+            "Authorization",
+            oauth1_request::post(
+                TWITTER_UPLOAD_MEDIA_URL,
+                &empty_request,
+                &token,
+                oauth1_request::HMAC_SHA1,
+            ),
+        )
+        .multipart(
+            Form::new()
+                .text("command", "FINALIZE")
+                .text("media_id", orig_media_id.media_id.to_string()),
+        )
+        .send()
+        .await?
+        .json::<UploadMediaResponse>()
+        .await?;
+
+    if let Some(p_info) = fin.processing_info {
+        if let Some(wait_sec) = p_info.check_after_secs {
+            debug!(
+                "Processing is not finished yet for ID {}, waiting {} secs",
+                orig_media_id.media_id, wait_sec
+            );
+            // getting here, we have a status and a check_after_secs
+            // this status can be anything but we will check it afterwards
+            // whatever happens, we can wait here before proceeding
+            sleep(Duration::from_secs(wait_sec)).await;
+
+            let command = UploadMediaCommand {
+                command: "STATUS".to_string(),
+                media_id: orig_media_id.media_id.to_string(),
+            };
+
+            loop {
+                debug!(
+                    "Checking on status for ID {} after waiting {} secs",
+                    orig_media_id.media_id, wait_sec
+                );
+
+                let status = client
+                    .get(TWITTER_UPLOAD_MEDIA_URL)
+                    .header(
+                        "Authorization",
+                        oauth1_request::get(
+                            TWITTER_UPLOAD_MEDIA_URL,
+                            &command,
+                            &token,
+                            oauth1_request::HMAC_SHA1,
+                        ),
+                    )
+                    .query(&command)
+                    .send()
+                    .await?
+                    .json::<UploadMediaResponse>()
+                    .await?;
+
+                let p_status = status.processing_info.unwrap(); // shouldn’t be None at this point
+                match p_status.state {
+                    UploadMediaResponseProcessingInfoState::Failed => {
+                        debug!("Processing has failed!");
+                        return Err(OolatoocsError::new(&format!(
+                            "Upload for {} (id: {}) has failed",
+                            u, orig_media_id.media_id
+                        ))
+                        .into());
+                    }
+                    UploadMediaResponseProcessingInfoState::Succeeded => {
+                        debug!("Processing has succeeded, exiting loop!");
+                        break;
+                    }
+                    UploadMediaResponseProcessingInfoState::Pending
+                    | UploadMediaResponseProcessingInfoState::InProgress => {
+                        debug!(
+                            "Processing still pending, waiting {} secs more…",
+                            p_status.check_after_secs.unwrap() // unwrap is safe here,
+                                                               // check_after_secs is only present
+                                                               // when status is pending or in
+                                                               // progress
+                        );
+                        sleep(Duration::from_secs(p_status.check_after_secs.unwrap())).await;
+                        continue;
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(orig_media_id.media_id)
 }
 
 /// This posts Tweets with all the associated medias
@@ -122,24 +374,32 @@ pub async fn post_tweet(
     config: &TwitterConfig,
     content: &str,
     medias: &[u64],
+    reply_to: &Option<u64>,
 ) -> Result<u64, Box<dyn Error>> {
-    let uri = "https://api.twitter.com/2/tweets";
     let empty_request = EmptyRequest {}; // Why? Because fuck you, that’s why!
     let token = get_token(config);
 
     let tweet = Tweet {
         text: content.to_string(),
-        media: TweetMediasIds {
+        media: medias.is_empty().not().then(|| TweetMediasIds {
             media_ids: medias.iter().map(|m| m.to_string()).collect(),
-        },
+        }),
+        reply: reply_to.map(|s| TweetReply {
+            in_reply_to_tweet_id: s.to_string(),
+        }),
     };
 
     let client = Client::new();
     let res = client
-        .post(uri)
+        .post(TWITTER_API_TWEET_URL)
         .header(
             "Authorization",
-            oauth1_request::post(uri, &empty_request, &token, oauth1_request::HMAC_SHA1),
+            oauth1_request::post(
+                TWITTER_API_TWEET_URL,
+                &empty_request,
+                &token,
+                oauth1_request::HMAC_SHA1,
+            ),
         )
         .json(&tweet)
         .send()