Merge branch 'feat/megalodon_1_1' into 'main'

⬆️: upgrade megalodon v1.1

See merge request veretcle/oolatoocs!38

.gitignore

@@ -2,3 +2,4 @@
 .last_tweet
 .config.toml
 .config.json
+.bsky.json

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "oolatoocs"
-version = "4.3.0"
+version = "4.4.1"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -12,7 +12,6 @@ env_logger = "^0.11"
 futures = "^0.3"
 html-escape = "^0.2"
 log = "^0.4"
-megalodon = "^1.0"
 oauth1-request = "^0.6"
 regex = "^1.10"
 reqwest = { version = "^0.12", features = ["json", "stream", "multipart"] }
@@ -24,6 +23,7 @@ bsky-sdk = "^0.1"
 atrium-api = { version = "^0.25", features = ["namespace-appbsky"] }
 image = "^0.25"
 webp = "^0.3"
+megalodon = "^1.1"
 
 [profile.release]
 strip = true

README.md

@@ -18,7 +18,7 @@ What it can do:
 * Cuts (poorly) the Toot in half in it’s too long for Bluesky and thread it (this is cut using a word count, not the best method, but it gets the job done);
 * Reuploads images/gifs/videos from Mastodon to Bluesky
     * ⚠️ Bluesky does not support mixing images and videos. You can have up to 4 images on a Bsky record **or** 1 video but not mix around. If you do so, only the video will be posted on Bluesky.
-    * ⚠️ Bluesky does not support images greater than 1Mb (that is 1,000,000,000 bytes or 976.6 KiB). I might incorporate soon a image quality reducer or WebP transcoding to avoid this issue.
+    * ⚠️ Bluesky does not support images greater than 1Mb (that is 1,000,000 bytes or 976.6 KiB), so Oolatoocs converts the image to WebP and progressively reduces the quality to fit that limitation.
 * Can reproduce threads from Mastodon to Bluesky
 * ⚠️ Bluesky does support polls for now. So the poll itself is just presented as text from Mastodon instead which is not the most elegant.
 * Can prevent a Toot from being recorded to Bluesky by using the #NoTweet (case-insensitive) hashtag in Mastodon
@@ -30,6 +30,7 @@ The configuration is relatively easy to follow:
 ```toml
 [oolatoocs]
 db_path = "/var/lib/oolatoocs/db.sqlite3" # the path to the DB where toots/tweets/records are stored
+remove_hashtags = false  # optional, default to false
 
 [mastodon] # This part can be generated, see below
 base = "https://m.nintendojo.fr"

src/bsky.rs

@@ -148,6 +148,31 @@ async fn get_record(
     Ok(record)
 }
 
+/// Generate an quote embed record into Bsky
+pub async fn generate_quote_records(
+    config: &BlueskyConfig,
+    quote_id: &str,
+) -> Option<atrium_api::types::Union<atrium_api::app::bsky::feed::post::RecordEmbedRefs>> {
+    // if we can’t match the quote_id, simply return None
+    let quote_record = match get_record(&config.handle, &rkey(quote_id)).await {
+        Ok(a) => a,
+        Err(_) => return None,
+    };
+
+    Some(atrium_api::types::Union::Refs(
+        atrium_api::app::bsky::feed::post::RecordEmbedRefs::AppBskyEmbedRecordMain(Box::new(
+            atrium_api::app::bsky::embed::record::MainData {
+                record: atrium_api::com::atproto::repo::strong_ref::MainData {
+                    cid: quote_record.data.cid.unwrap(),
+                    uri: quote_record.data.uri.to_owned(),
+                }
+                .into(),
+            }
+            .into(),
+        )),
+    ))
+}
+
 /// Generate an embed card record into Bsky
 /// If the preview image does not exist or fails to upload, it is simply ignored
 pub async fn generate_embed_records(
@@ -200,7 +225,7 @@ pub async fn generate_media_records(
         .collect();
     let video_media_attach: Vec<_> = media_attach
         .iter()
-        .filter(|x| (x.r#type == AttachmentType::Video || x.r#type == AttachmentType::Gifv))
+        .filter(|x| x.r#type == AttachmentType::Video || x.r#type == AttachmentType::Gifv)
         .cloned()
         .collect();
 

src/lib.rs

@@ -19,7 +19,8 @@ use utils::{generate_multi_tweets, strip_everything};
 
 mod bsky;
 use bsky::{
-    build_post_record, generate_embed_records, generate_media_records, get_session, BskyReply,
+    build_post_record, generate_embed_records, generate_media_records, generate_quote_records,
+    get_session, BskyReply,
 };
 
 use rusqlite::Connection;
@@ -93,7 +94,9 @@ pub async fn run(config: &Config) {
                 true => toot.tags.clone(),
                 false => vec![],
             };
-        let Ok(mut tweet_content) = strip_everything(&toot.content, &toot_tags) else {
+        let Ok(mut tweet_content) =
+            strip_everything(&toot.content, &toot_tags, &config.mastodon.base)
+        else {
             continue; // skip in case we can’t strip something
         };
 
@@ -158,15 +161,25 @@ pub async fn run(config: &Config) {
             });
         };
 
-        // treats medias
-        let mut record_embed = generate_media_records(&bluesky, &toot.media_attachments).await;
+        // Don’t know how to union things so…
+        // cards have the least priority (you cannot embed card and images anyway)
+        // images have a higher priority
+        // quotes have the highest priority
 
-        // treats embed cards if any
-        if let Some(card) = &toot.card {
-            if record_embed.is_none() {
-                record_embed = generate_embed_records(&bluesky, card).await;
-            }
+        let record_embed = if toot.reblog.is_some() {
+            let quote_record =
+                read_state(&conn, Some(toot.reblog.unwrap().id.parse::<u64>().unwrap()));
+            match quote_record {
+                Ok(Some(q)) => generate_quote_records(&config.bluesky, &q.record_uri).await,
+                _ => None,
             }
+        } else if toot.media_attachments.len() > usize::from(0u8) {
+            generate_media_records(&bluesky, &toot.media_attachments).await
+        } else if toot.card.is_some() {
+            generate_embed_records(&bluesky, &toot.card.unwrap()).await
+        } else {
+            None
+        };
 
         // posts corresponding tweet
         let record = build_post_record(

src/mastodon.rs

@@ -1,7 +1,7 @@
 use crate::config::MastodonConfig;
 use chrono::{DateTime, Utc};
 use megalodon::{
-    entities::{Status, StatusVisibility},
+    entities::{QuotedStatus, Status, StatusVisibility},
     generator,
     mastodon::mastodon::Mastodon,
     megalodon::AppInputOptions,
@@ -55,9 +55,19 @@ pub async fn get_mastodon_timeline_since(
                     .clone()
                     .is_some_and(|r| r == t.account.id)
         })
-        .filter(|t| t.visibility == StatusVisibility::Public) // excludes everything that isn’t
-        // public
-        .filter(|t| t.reblog.is_none()) // excludes reblogs
+        .filter(|t| t.visibility == StatusVisibility::Public) // excludes everything that isn’t public
+        .filter(|t| t.reblog.is_none()) // exclude reblogs
+        .filter(|t| {
+            // exclude quotes that aren’t ours
+            t.quote.is_none()
+                || t.quote.clone().is_some_and(|r| match r {
+                    QuotedStatus::Quote(q) => q
+                        .quoted_status
+                        .clone()
+                        .is_some_and(|iq| iq.account.id == t.account.id),
+                    _ => false,
+                })
+        })
         .cloned()
         .collect();
 

src/utils.rs

@@ -53,10 +53,16 @@ fn twitter_count(content: &str) -> usize {
     count
 }
 
-pub fn strip_everything(content: &str, tags: &Vec<Tag>) -> Result<String, Box<dyn Error>> {
+pub fn strip_everything(
+    content: &str,
+    tags: &Vec<Tag>,
+    mastodon_base: &str,
+) -> Result<String, Box<dyn Error>> {
     let mut res = strip_html_tags(&content.replace("</p><p>", "\n\n").replace("<br />", "\n"));
 
-    strip_mastodon_tags(&mut res, tags).unwrap();
+    strip_quote_header(&mut res, mastodon_base)?;
+
+    strip_mastodon_tags(&mut res, tags)?;
 
     res = res.trim_end_matches('\n').trim_end_matches(' ').to_string();
     res = decode_html_entities(&res).to_string();
@@ -64,6 +70,16 @@ pub fn strip_everything(content: &str, tags: &Vec<Tag>) -> Result<String, Box<dy
     Ok(res)
 }
 
+fn strip_quote_header(content: &mut String, mastodon_base: &str) -> Result<(), Box<dyn Error>> {
+    let re = Regex::new(&format!(
+        r"^RE: {}\S+\n\n",
+        mastodon_base.replace(".", r"\.")
+    ))?;
+    *content = re.replace(content, "").to_string();
+
+    Ok(())
+}
+
 fn strip_mastodon_tags(content: &mut String, tags: &Vec<Tag>) -> Result<(), Box<dyn Error>> {
     for tag in tags {
         let re = Regex::new(&format!("(?i)(#{} ?)", &tag.name))?;
@@ -186,9 +202,19 @@ mod tests {
 
     #[test]
     fn test_strip_everything() {
+        // a classic toot
         let content = "<p>Ce soir à 21h, c&#39;est le Dojobar ! Au programme ce soir, une rétrospective sur la série Mario &amp; Luigi.<br />Comme d&#39;hab, le Twitch sera ici : <a href=\"https://twitch.tv/nintendojofr\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">twitch.tv/nintendojofr</span><span class=\"invisible\"></span></a><br />Ou juste l&#39;audio là : <a href=\"https://nintendojo.fr/dojobar\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">nintendojo.fr/dojobar</span><span class=\"invisible\"></span></a><br />A toute !</p>";
         let expected_result = "Ce soir à 21h, c'est le Dojobar ! Au programme ce soir, une rétrospective sur la série Mario & Luigi.\nComme d'hab, le Twitch sera ici : https://twitch.tv/nintendojofr\nOu juste l'audio là : https://nintendojo.fr/dojobar\nA toute !".to_string();
-        let result = strip_everything(content, &vec![]).unwrap();
+        let result = strip_everything(content, &vec![], "https://m.nintendojo.fr").unwrap();
+
+        assert_eq!(result, expected_result);
+
+        // a quoted toot
+        let content = "<p class=\"quote-inline\">RE: <a href=\"https://m.nintendojo.fr/@nintendojofr/115446347351491651\" target=\"_blank\" rel=\"nofollow noopener\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">m.nintendojo.fr/@nintendojofr/</span><span class=\"invisible\">115446347351491651</span></a></p><p>Assassin’s Creed Shadows pèsera environ 62,8 Go sur Switch 2 (et un peu plus de 100 Go sur les autres supports), soit tout juste pour rentrer sur une cartouche de 64 Go.</p><p>Ou pas, pour rappel…</p><p><a href=\"https://m.nintendojo.fr/tags/AssassinsCreedShadows\" class=\"mention hashtag\" rel=\"tag\">#<span>AssassinsCreedShadows</span></a> <a href=\"https://m.nintendojo.fr/tags/Ubisoft\" class=\"mention hashtag\" rel=\"tag\">#<span>Ubisoft</span></a> <a href=\"https://m.nintendojo.fr/tags/NintendoSwitch2\" class=\"mention hashtag\" rel=\"tag\">#<span>NintendoSwitch2</span></a></p>";
+
+        let expected_result = "Assassin’s Creed Shadows pèsera environ 62,8 Go sur Switch 2 (et un peu plus de 100 Go sur les autres supports), soit tout juste pour rentrer sur une cartouche de 64 Go.\n\nOu pas, pour rappel…\n\n#AssassinsCreedShadows #Ubisoft #NintendoSwitch2";
+
+        let result = strip_everything(content, &vec![], "https://m.nintendojo.fr").unwrap();
 
         assert_eq!(result, expected_result);
     }