Merge branch 'master' into 'noasync'

# Conflicts:
#   .gitlab-ci.yml
#   Cargo.lock
#   Cargo.toml
#   src/lib.rs

.gitlab-ci.yml

@@ -9,5 +9,5 @@ rust-latest:
     image: rust:latest
     script:
         - cargo build --release --verbose
-        - strip target/release/scootaloo
+        - strip target/release/${CI_PROJECT_NAME}
 

CHANGELOG

@@ -1,3 +1,23 @@
+# v0.3.3
+
+* optimizing the size of the final executable (now ⩽ 6MiB)
+
+# v0.3.2
+
+* 100% async version
+* now media are download in parallel thanks to async
+* log are introduced into code for your viewing pleasure
+
+# v0.2.3
+
+* using the async version of `reqwest`
+* introducing async functions and make `tokio` the de facto executor for everything async
+
+# v0.2.1
+
+* using `tokio-compat` to avoid having 3 different versions of `tokio` in the same executable
+* encapsulating async calls inside blocking tokio runtime calls
+
 # v0.1.8
 
 * fix #1: mentions are treated like decoded urls (this is not really needed to push it this far but it would be easier in case you want to modify it)

Cargo.lock

@@ -94,6 +94,12 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "base64"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -140,9 +146,15 @@ dependencies = [
 
 [[package]]
 name = "bytes"
-version = "0.5.4"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1"
+checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
+
+[[package]]
+name = "bytes"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
 
 [[package]]
 name = "bytes"
@@ -349,7 +361,7 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6"
 dependencies = [
- "generic-array",
+ "generic-array 0.14.4",
  "subtle",
 ]
 
@@ -376,7 +388,16 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
 dependencies = [
- "generic-array",
+ "generic-array 0.12.3",
+]
+
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array 0.14.4",
 ]
 
 [[package]]
@@ -411,7 +432,7 @@ dependencies = [
  "regex",
  "serde",
  "serde_json",
- "sha-1",
+ "sha-1 0.9.4",
  "thiserror",
  "tokio 1.5.0",
  "url 2.2.1",
@@ -425,9 +446,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
 
 [[package]]
 name = "elefren"
-version = "0.20.1"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfca8d8d0147086081224e22183a37a7b98e3230b945a717f1b5a0eed5fb07af"
+checksum = "ba08a959d3824df696d49c2ec023f45851f663c47b57b2de933aab749104cd18"
 dependencies = [
  "chrono",
  "doc-comment",
@@ -443,6 +464,7 @@ dependencies = [
  "skeptic",
  "tap-reader",
  "try_from",
+ "tungstenite",
  "url 1.7.2",
 ]
 
@@ -673,6 +695,16 @@ dependencies = [
  "version_check 0.9.1",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
+dependencies = [
+ "typenum",
+ "version_check 0.9.1",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.1.14"
@@ -760,7 +792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
 dependencies = [
  "crypto-mac",
- "digest",
+ "digest 0.9.0",
 ]
 
 [[package]]
@@ -786,7 +818,7 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b"
 dependencies = [
- "bytes 0.5.4",
+ "bytes 0.5.6",
  "fnv",
  "itoa",
 ]
@@ -826,6 +858,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9"
 
+[[package]]
+name = "httpdate"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47"
+
 [[package]]
 name = "hyper"
 version = "0.12.35"
@@ -1114,9 +1152,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.6.21"
+version = "0.6.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
+checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4"
 dependencies = [
  "cfg-if 0.1.9",
  "fuchsia-zircon",
@@ -1146,9 +1184,9 @@ dependencies = [
 
 [[package]]
 name = "miow"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
+checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
 dependencies = [
  "kernel32-sys",
  "net2",
@@ -1185,9 +1223,9 @@ dependencies = [
 
 [[package]]
 name = "net2"
-version = "0.2.33"
+version = "0.2.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
+checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
 dependencies = [
  "cfg-if 0.1.9",
  "libc",
@@ -1203,6 +1241,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "ntapi"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a31937dea023539c72ddae0e3571deadc1414b300483fa7aaec176168cfa9d2"
+dependencies = [
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.42"
@@ -1244,6 +1291,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
 [[package]]
 name = "openssl"
 version = "0.10.33"
@@ -1328,6 +1381,20 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "parking_lot_core"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
+dependencies = [
+ "cfg-if 1.0.0",
+ "instant",
+ "libc",
+ "redox_syscall",
+ "smallvec 1.2.0",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "1.0.1"
@@ -1866,7 +1933,6 @@ dependencies = [
  "clap",
  "egg-mode",
  "elefren",
- "futures 0.3.5",
  "htmlescape",
  "log",
  "reqwest 0.11.3",
@@ -2076,6 +2142,17 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "string"
 version = "0.2.1"
@@ -2505,6 +2582,12 @@ dependencies = [
  "percent-encoding 2.1.0",
 ]
 
+[[package]]
+name = "utf-8"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"
+
 [[package]]
 name = "uuid"
 version = "0.7.4"
@@ -2721,6 +2804,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "winreg"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
+dependencies = [
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "ws2_32-sys"
 version = "0.2.1"

README.md

@@ -3,7 +3,7 @@ A Twitter to Mastodon copy bot written in Rust
 It:
 * copies the content (text) of the original Tweet
 * dereferences the links
-* gets every attach media (photo, video or gif)
+* gets every attached media (photo, video or gif)
 
 If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
 
@@ -74,4 +74,3 @@ echo -n '8189881949849' > last_tweet
 
 **This file should only contain the last tweet ID without any other char (no EOL or new line).**
 
-Oh and everything is sync (and not async) so this does not run at a blazing speed…

src/lib.rs

@@ -6,6 +6,7 @@ use std::{
     fmt,
     fs::{read_to_string, write},
     error::Error,
+    sync::{Arc, Mutex},
 };
 
 // toml
@@ -52,7 +53,7 @@ use log::{info, warn, error, debug};
 /*
  * Those functions are related to the Twitter side of things
  */
-/// Read last tweet id from a file
+/// Reads last tweet id from a file
 fn read_state(s: &str) -> Option<u64> {
     let state = read_to_string(s);
 
@@ -69,7 +70,7 @@ fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
     write(f, format!("{}", s))
 }
 
-/// Gets twitter oauth2 token
+/// Gets Twitter oauth2 token
 fn get_oauth2_token(config: &Config) -> Token {
     let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
     let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
@@ -80,12 +81,13 @@ fn get_oauth2_token(config: &Config) -> Token {
     }
 }
 
-/// Gets twitter user timeline
+/// Gets Twitter user timeline
 async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
     // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_timeline, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
+    let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
                 .with_page_size(200)
-                .older(lid).await?;
+                .older(lid)
+                .await?;
 
     Ok(feed.to_vec())
 }
@@ -206,6 +208,15 @@ async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
     Ok(dest_filepath)
 }
 
+/**********
+ * This is the struct that holds the Mastodon Media ID and the Twitter Media URL at the same Time
+**********/
+#[derive(Debug)]
+struct ScootalooSpawnResponse {
+    mastodon_media_id: String,
+    twitter_media_url: String,
+}
+
 /**********
  * local error handler
 **********/
@@ -324,10 +335,12 @@ pub async fn run(config: Config) {
     let token = get_oauth2_token(&config);
 
     // get Mastodon instance
-    let mastodon = get_mastodon_token(&config.mastodon);
+    let mastodon = Arc::new(Mutex::new(get_mastodon_token(&config.mastodon)));
 
     // get user timeline feed (Vec<tweet>)
-    let mut feed = get_user_timeline(&config, token, last_tweet_id).await.unwrap_or_else(|e|
+    let mut feed = get_user_timeline(&config, token, last_tweet_id)
+        .await
+        .unwrap_or_else(|e|
         panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
     );
 
@@ -341,6 +354,7 @@ pub async fn run(config: Config) {
     feed.reverse();
 
     for tweet in &feed {
+        debug!("Treating Tweet {} inside feed", tweet.id);
         // determine if the tweet is part of a thread (response to self) or a standard response
         debug!("Treating Tweet {} inside feed", tweet.id);
         if let Some(r) =  &tweet.in_reply_to_screen_name {
@@ -364,6 +378,8 @@ pub async fn run(config: Config) {
 
         // reupload the attachments if any
         if let Some(m) = &tweet.extended_entities {
+            let (tx, mut rx) = mpsc::channel(4);
+
             for media in &m.media {
                 let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path).await {
                     Ok(m) => m,
@@ -390,6 +406,7 @@ pub async fn run(config: Config) {
                 status_text = status_text.replace(&media.url, "");
             }
         }
+        // finished reuploading attachments, now let’s do the toot baby!
 
         debug!("Building corresponding Mastodon status");
         let status = StatusBuilder::new()
@@ -399,7 +416,8 @@ pub async fn run(config: Config) {
             .expect(&format!("Cannot build status with text {}", &status_text));
 
         // publish status
-        mastodon.new_status(status).unwrap();
+        // again unwrap is safe here as we are in the main thread
+        mastodon.lock().unwrap().new_status(status).unwrap();
         // this will panic if it cannot publish the status, which is a good thing, it allows the
         // last_tweet gathered not to be written