diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5061a23..3a2c988 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,5 +9,5 @@ rust-latest: image: rust:latest script: - cargo build --release --verbose - - strip target/release/scootaloo + - strip target/release/${CI_PROJECT_NAME} diff --git a/CHANGELOG b/CHANGELOG index ddd1a49..a6d432b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,23 @@ +# v0.3.3 + +* optimizing the size of the final executable (now ⩽ 6MiB) + +# v0.3.2 + +* 100% async version +* now media are download in parallel thanks to async +* log are introduced into code for your viewing pleasure + +# v0.2.3 + +* using the async version of `reqwest` +* introducing async functions and make `tokio` the de facto executor for everything async + +# v0.2.1 + +* using `tokio-compat` to avoid having 3 different versions of `tokio` in the same executable +* encapsulating async calls inside blocking tokio runtime calls + # v0.1.8 * fix #1: mentions are treated like decoded urls (this is not really needed to push it this far but it would be easier in case you want to modify it) diff --git a/Cargo.lock b/Cargo.lock index c5a4d8c..661241e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,6 +94,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.2.1" @@ -140,9 +146,15 @@ dependencies = [ [[package]] name = "bytes" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "bytes" @@ -349,7 +361,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ - "generic-array", + "generic-array 0.14.4", "subtle", ] @@ -376,7 +388,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", ] [[package]] @@ -411,7 +432,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sha-1", + "sha-1 0.9.4", "thiserror", "tokio 1.5.0", "url 2.2.1", @@ -425,9 +446,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" [[package]] name = "elefren" -version = "0.20.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfca8d8d0147086081224e22183a37a7b98e3230b945a717f1b5a0eed5fb07af" +checksum = "ba08a959d3824df696d49c2ec023f45851f663c47b57b2de933aab749104cd18" dependencies = [ "chrono", "doc-comment", @@ -443,6 +464,7 @@ dependencies = [ "skeptic", "tap-reader", "try_from", + "tungstenite", "url 1.7.2", ] @@ -673,6 +695,16 @@ dependencies = [ "version_check 0.9.1", ] +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check 0.9.1", +] + [[package]] name = "getrandom" version = "0.1.14" @@ -760,7 +792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ "crypto-mac", - "digest", + "digest 0.9.0", ] [[package]] @@ -786,7 +818,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b" dependencies = [ - "bytes 0.5.4", + "bytes 0.5.6", "fnv", "itoa", ] @@ -826,6 +858,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + [[package]] name = "hyper" version = "0.12.35" @@ -1114,9 +1152,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.21" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" dependencies = [ "cfg-if 0.1.9", "fuchsia-zircon", @@ -1146,9 +1184,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" dependencies = [ "kernel32-sys", "net2", @@ -1185,9 +1223,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.33" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" dependencies = [ "cfg-if 0.1.9", "libc", @@ -1203,6 +1241,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ntapi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a31937dea023539c72ddae0e3571deadc1414b300483fa7aaec176168cfa9d2" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "num-integer" version = "0.1.42" @@ -1244,6 +1291,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.33" @@ -1328,6 +1381,20 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec 1.2.0", + "winapi 0.3.8", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -1866,7 +1933,6 @@ dependencies = [ "clap", "egg-mode", "elefren", - "futures 0.3.5", "htmlescape", "log", "reqwest 0.11.3", @@ -2076,6 +2142,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.8", +] + [[package]] name = "string" version = "0.2.1" @@ -2505,6 +2582,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "utf-8" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" + [[package]] name = "uuid" version = "0.7.4" @@ -2721,6 +2804,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/README.md b/README.md index 65c9986..ed5b972 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A Twitter to Mastodon copy bot written in Rust It: * copies the content (text) of the original Tweet * dereferences the links -* gets every attach media (photo, video or gif) +* gets every attached media (photo, video or gif) If any of the last steps failed, the Toot gets published with the exact same text as the Tweet. @@ -74,4 +74,3 @@ echo -n '8189881949849' > last_tweet **This file should only contain the last tweet ID without any other char (no EOL or new line).** -Oh and everything is sync (and not async) so this does not run at a blazing speed… diff --git a/src/lib.rs b/src/lib.rs index f06de06..df88ce3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ use std::{ fmt, fs::{read_to_string, write}, error::Error, + sync::{Arc, Mutex}, }; // toml @@ -52,7 +53,7 @@ use log::{info, warn, error, debug}; /* * Those functions are related to the Twitter side of things */ -/// Read last tweet id from a file +/// Reads last tweet id from a file fn read_state(s: &str) -> Option { let state = read_to_string(s); @@ -69,7 +70,7 @@ fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> { write(f, format!("{}", s)) } -/// Gets twitter oauth2 token +/// Gets Twitter oauth2 token fn get_oauth2_token(config: &Config) -> Token { let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret)); let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret)); @@ -80,12 +81,13 @@ fn get_oauth2_token(config: &Config) -> Token { } } -/// Gets twitter user timeline +/// Gets Twitter user timeline async fn get_user_timeline(config: &Config, token: Token, lid: Option) -> Result, Box> { // fix the page size to 200 as it is the maximum Twitter authorizes - let (_timeline, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token) + let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token) .with_page_size(200) - .older(lid).await?; + .older(lid) + .await?; Ok(feed.to_vec()) } @@ -206,6 +208,15 @@ async fn cache_media(u: &str, t: &str) -> Result> { Ok(dest_filepath) } +/********** + * This is the struct that holds the Mastodon Media ID and the Twitter Media URL at the same Time +**********/ +#[derive(Debug)] +struct ScootalooSpawnResponse { + mastodon_media_id: String, + twitter_media_url: String, +} + /********** * local error handler **********/ @@ -324,10 +335,12 @@ pub async fn run(config: Config) { let token = get_oauth2_token(&config); // get Mastodon instance - let mastodon = get_mastodon_token(&config.mastodon); + let mastodon = Arc::new(Mutex::new(get_mastodon_token(&config.mastodon))); // get user timeline feed (Vec) - let mut feed = get_user_timeline(&config, token, last_tweet_id).await.unwrap_or_else(|e| + let mut feed = get_user_timeline(&config, token, last_tweet_id) + .await + .unwrap_or_else(|e| panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e) ); @@ -341,6 +354,7 @@ pub async fn run(config: Config) { feed.reverse(); for tweet in &feed { + debug!("Treating Tweet {} inside feed", tweet.id); // determine if the tweet is part of a thread (response to self) or a standard response debug!("Treating Tweet {} inside feed", tweet.id); if let Some(r) = &tweet.in_reply_to_screen_name { @@ -364,6 +378,8 @@ pub async fn run(config: Config) { // reupload the attachments if any if let Some(m) = &tweet.extended_entities { + let (tx, mut rx) = mpsc::channel(4); + for media in &m.media { let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path).await { Ok(m) => m, @@ -390,6 +406,7 @@ pub async fn run(config: Config) { status_text = status_text.replace(&media.url, ""); } } + // finished reuploading attachments, now let’s do the toot baby! debug!("Building corresponding Mastodon status"); let status = StatusBuilder::new() @@ -399,7 +416,8 @@ pub async fn run(config: Config) { .expect(&format!("Cannot build status with text {}", &status_text)); // publish status - mastodon.new_status(status).unwrap(); + // again unwrap is safe here as we are in the main thread + mastodon.lock().unwrap().new_status(status).unwrap(); // this will panic if it cannot publish the status, which is a good thing, it allows the // last_tweet gathered not to be written