Merge branch '15-feat-make-hashtags-removal-optional' into 'main'

✨: hashtag removal is now optional

Closes #15

See merge request veretcle/oolatoocs!34

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "oolatoocs"
-version = "4.1.3"
+version = "4.3.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -8,20 +8,20 @@ edition = "2021"
 [dependencies]
 chrono = "^0.4"
 clap = "^4"
-env_logger = "^0.10"
+env_logger = "^0.11"
 futures = "^0.3"
 html-escape = "^0.2"
 log = "^0.4"
 megalodon = "^1.0"
 oauth1-request = "^0.6"
 regex = "^1.10"
-reqwest = { version = "^0.11", features = ["json", "stream", "multipart"] }
+reqwest = { version = "^0.12", features = ["json", "stream", "multipart"] }
-rusqlite = { version = "^0.30", features = ["chrono"] }
+rusqlite = { version = "^0.33", features = ["chrono"] }
 serde = { version = "^1.0", features = ["derive"] }
 tokio = { version = "^1.33", features = ["rt-multi-thread", "macros"] }
 toml = "^0.8"
 bsky-sdk = "^0.1"
-atrium-api = "^0.24"
+atrium-api = { version = "^0.25", features = ["namespace-appbsky"] }
 image = "^0.25"
 webp = "^0.3"
 

src/bsky.rs

@@ -1,7 +1,7 @@
 use crate::config::BlueskyConfig;
 use atrium_api::{
     app::bsky::feed::post::RecordData, com::atproto::repo::upload_blob::Output,
-    types::string::Datetime, types::string::Language,
+    types::string::Datetime, types::string::Language, types::string::RecordKey,
 };
 use bsky_sdk::{
     agent::config::{Config, FileStore},
@@ -11,7 +11,10 @@ use bsky_sdk::{
 use futures::{stream, StreamExt};
 use image::ImageReader;
 use log::{debug, error, warn};
-use megalodon::entities::attachment::{Attachment, AttachmentType};
+use megalodon::entities::{
+    attachment::{Attachment, AttachmentType},
+    card::Card,
+};
 use regex::Regex;
 use std::{error::Error, fs::exists, io::Cursor};
 use webp::*;
@@ -136,7 +139,7 @@ async fn get_record(
                 cid: None,
                 collection: atrium_api::types::string::Nsid::new("app.bsky.feed.post".to_string())?,
                 repo: atrium_api::types::string::Handle::new(config.to_string())?.into(),
-                rkey: rkey.to_string(),
+                rkey: RecordKey::new(rkey.to_string())?,
             }
             .into(),
         )
@@ -145,7 +148,43 @@ async fn get_record(
     Ok(record)
 }
 
-// it’s ugly af but it gets the job done for now
+/// Generate an embed card record into Bsky
+/// If the preview image does not exist or fails to upload, it is simply ignored
+pub async fn generate_embed_records(
+    bsky: &BskyAgent,
+    card: &Card,
+) -> Option<atrium_api::types::Union<atrium_api::app::bsky::feed::post::RecordEmbedRefs>> {
+    // uploads the image card, if it fails, simply ignore everything
+    let blob = if let Some(url) = &card.image {
+        if let Ok(image_blob) = upload_media(true, bsky, url).await {
+            Some(image_blob.blob.clone())
+        } else {
+            None
+        }
+    } else {
+        None
+    };
+
+    let record_card = atrium_api::app::bsky::embed::external::ExternalData {
+        description: card.description.clone(),
+        thumb: blob,
+        title: card.title.clone(),
+        uri: card.url.clone(),
+    };
+
+    Some(atrium_api::types::Union::Refs(
+        atrium_api::app::bsky::feed::post::RecordEmbedRefs::AppBskyEmbedExternalMain(Box::new(
+            atrium_api::app::bsky::embed::external::MainData {
+                external: record_card.into(),
+            }
+            .into(),
+        )),
+    ))
+}
+
+/// Generate an array of Bsky media records
+/// As Bsky does not support multiple video in a record or mix of video and images, video has the
+/// highest priority
 pub async fn generate_media_records(
     bsky: &BskyAgent,
     media_attach: &[Attachment],
@@ -239,11 +278,20 @@ async fn upload_media(
     } else {
         // this is an image and it’s over 1Mb long
         debug!("Img file too large: {}", content_length);
+        // defaults to 95% quality for WebP compression
+        let mut default_quality = 95f32;
         let img = ImageReader::new(Cursor::new(dl.bytes().await?))
             .with_guessed_format()?
             .decode()?;
         let encoder: Encoder = Encoder::from_image(&img)?;
-        let webp: WebPMemory = encoder.encode(90f32);
+        let mut webp: WebPMemory = encoder.encode(default_quality);
+
+        while webp.len() > 1_000_000 {
+            debug!("Img file too large at {}%, reducing…", default_quality);
+            default_quality -= 5.0;
+            webp = encoder.encode(default_quality);
+        }
+
         webp.to_vec()
     };
 

src/config.rs

@@ -11,6 +11,17 @@ pub struct Config {
 #[derive(Debug, Deserialize)]
 pub struct OolatoocsConfig {
     pub db_path: String,
+    #[serde(default)]
+    pub remove_hashtags: bool,
+}
+
+impl Default for OolatoocsConfig {
+    fn default() -> Self {
+        OolatoocsConfig {
+            db_path: "/var/lib/oolatoocs/db".to_string(),
+            remove_hashtags: false,
+        }
+    }
 }
 
 #[derive(Debug, Deserialize)]

src/lib.rs

@@ -18,7 +18,9 @@ mod utils;
 use utils::{generate_multi_tweets, strip_everything};
 
 mod bsky;
-use bsky::{build_post_record, generate_media_records, get_session, BskyReply};
+use bsky::{
+    build_post_record, generate_embed_records, generate_media_records, get_session, BskyReply,
+};
 
 use rusqlite::Connection;
 
@@ -86,7 +88,12 @@ pub async fn run(config: &Config) {
         }
 
         // form tweet_content and strip everything useless in it
-        let Ok(mut tweet_content) = strip_everything(&toot.content, &toot.tags) else {
+        let toot_tags: Vec<megalodon::entities::status::Tag> =
+            match &config.oolatoocs.remove_hashtags {
+                true => toot.tags.clone(),
+                false => vec![],
+            };
+        let Ok(mut tweet_content) = strip_everything(&toot.content, &toot_tags) else {
             continue; // skip in case we can’t strip something
         };
 
@@ -152,14 +159,21 @@ pub async fn run(config: &Config) {
         };
 
         // treats medias
-        let record_medias = generate_media_records(&bluesky, &toot.media_attachments).await;
+        let mut record_embed = generate_media_records(&bluesky, &toot.media_attachments).await;
+
+        // treats embed cards if any
+        if let Some(card) = &toot.card {
+            if record_embed.is_none() {
+                record_embed = generate_embed_records(&bluesky, card).await;
+            }
+        }
 
         // posts corresponding tweet
         let record = build_post_record(
             &config.bluesky,
             &tweet_content,
             &toot.language,
-            record_medias,
+            record_embed,
             &record_reply_to,
         )
         .await

src/state.rs

@@ -82,10 +82,7 @@ pub fn write_state(conn: &Connection, t: TootRecord) -> Result<(), Box<dyn Error
 
 /// Initiates the DB from path
 pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
-    debug!(
+    debug!("Initializing DB for {}", env!("CARGO_PKG_NAME"));
-        "{}",
-        format!("Initializing DB for {}", env!("CARGO_PKG_NAME"))
-    );
     let conn = Connection::open(d)?;
 
     conn.execute(

src/utils.rs

@@ -38,7 +38,13 @@ fn twitter_count(content: &str) -> usize {
 
     for word in split_content {
         if word.starts_with("http://") || word.starts_with("https://") {
-            count += 23;
+            // It’s not that simple. Bsky adapts itself to the URL.
+            // https://github.com -> 10 chars
+            // https://github.com/ -> 10 chars
+            // https://github.com/NVNTLabs -> 19 chars
+            // https://github.com/NVNTLabs/ -> 20 chars
+            // so taking the maximum here to simplify things
+            count += 26;
         } else {
             count += word.chars().count();
         }
@@ -100,11 +106,11 @@ mod tests {
 
         let content = "Shoot out to https://y.ml/ !";
 
-        assert_eq!(twitter_count(content), 38);
+        assert_eq!(twitter_count(content), 41);
 
         let content = "this is the link https://www.google.com/tamerelol/youpi/tonperemdr/tarace.html if you like! What if I shit a final";
 
-        assert_eq!(twitter_count(content), 76);
+        assert_eq!(twitter_count(content), 79);
 
         let content = "multi  ple     space";
 
@@ -112,7 +118,7 @@ mod tests {
 
         let content = "This link is LEEEEET\n\nhttps://www.factornews.com/actualites/ca-sent-le-sapin-pour-free-radical-design-49985.html";
 
-        assert_eq!(twitter_count(content), 45);
+        assert_eq!(twitter_count(content), 48);
     }
 
     #[test]