diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 622893d..9cd248c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,5 +6,7 @@ rust-latest: image: rust:latest script: - cargo build --verbose - - cargo test --verbose + - cargo build --release --verbose + - strip target/release/${CI_PROJECT_NAME} + - du -h target/release/${CI_PROJECT_NAME} diff --git a/Cargo.lock b/Cargo.lock index ae8cfaa..56463dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,12 +94,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -265,6 +259,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "colored" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" +dependencies = [ + "atty", + "lazy_static", + "winapi 0.3.8", +] + [[package]] name = "cookie" version = "0.12.0" @@ -373,9 +378,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ "generic-array 0.14.4", "subtle", @@ -431,27 +436,26 @@ checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" [[package]] name = "egg-mode" version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f877bc908325f50163ff1670a4733eabf87942511ccfd907fbef4e239c3f8aa" +source = "git+https://github.com/egg-mode-rs/egg-mode?rev=6b81073eba9c3b123ca0e80bdb5ef61d1758f131#6b81073eba9c3b123ca0e80bdb5ef61d1758f131" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "chrono", "derive_more", "futures 0.3.5", "hmac", - "hyper 0.13.2", - "hyper-tls 0.4.1", + "hyper 0.14.4", + "hyper-tls 0.5.0", "lazy_static", "mime", "native-tls", "percent-encoding 2.1.0", - "rand 0.7.3", + "rand 0.8.3", "regex", "serde", "serde_json", "sha-1 0.9.4", "thiserror", - "tokio 0.2.25", + "tokio 1.3.0", "url 2.2.1", ] @@ -735,7 +739,18 @@ checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ "cfg-if 0.1.9", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -762,25 +777,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "h2" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9433d71e471c1736fd5a61b671fc0b148d7a2992f666c958d03cd8feb3b88d1" -dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.0", - "indexmap", - "log", - "slab", - "tokio 0.2.25", - "tokio-util 0.2.0", -] - [[package]] name = "h2" version = "0.3.1" @@ -796,7 +792,7 @@ dependencies = [ "indexmap", "slab", "tokio 1.3.0", - "tokio-util 0.6.4", + "tokio-util", "tracing", ] @@ -817,9 +813,9 @@ dependencies = [ [[package]] name = "hmac" -version = "0.8.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ "crypto-mac", "digest 0.9.0", @@ -865,16 +861,6 @@ dependencies = [ "tokio-buf", ] -[[package]] -name = "http-body" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" -dependencies = [ - "bytes 0.5.6", - "http 0.2.0", -] - [[package]] name = "http-body" version = "0.4.0" @@ -927,30 +913,6 @@ dependencies = [ "want 0.2.0", ] -[[package]] -name = "hyper" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1c527bbc634be72aa7ba31e4e4def9bbb020f5416916279b7c705cd838893e" -dependencies = [ - "bytes 0.5.6", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.2.1", - "http 0.2.0", - "http-body 0.3.1", - "httparse", - "itoa", - "log", - "net2", - "pin-project 0.4.8", - "time", - "tokio 0.2.25", - "tower-service", - "want 0.3.0", -] - [[package]] name = "hyper" version = "0.14.4" @@ -1005,19 +967,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "hyper-tls" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" -dependencies = [ - "bytes 0.5.6", - "hyper 0.13.2", - "native-tls", - "tokio 0.2.25", - "tokio-tls", -] - [[package]] name = "hyper-tls" version = "0.5.0" @@ -1072,6 +1021,15 @@ dependencies = [ "bytes 0.5.6", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -1150,6 +1108,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.8" @@ -1389,11 +1356,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.3.3", + "parking_lot_core 0.6.2", "rustc_version", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api 0.4.3", + "parking_lot_core 0.8.2", +] + [[package]] name = "parking_lot_core" version = "0.6.2" @@ -1409,6 +1387,20 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec 1.2.0", + "winapi 0.3.8", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -1499,12 +1491,6 @@ dependencies = [ "syn", ] -[[package]] -name = "pin-project-lite" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" - [[package]] name = "pin-project-lite" version = "0.2.6" @@ -1525,9 +1511,9 @@ checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "proc-macro-hack" @@ -1619,13 +1605,25 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.14", "libc", "rand_chacha 0.2.1", "rand_core 0.5.1", "rand_hc 0.2.0", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + [[package]] name = "rand_chacha" version = "0.1.1" @@ -1646,6 +1644,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -1667,7 +1675,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.14", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", ] [[package]] @@ -1688,6 +1705,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + [[package]] name = "rand_isaac" version = "0.1.1" @@ -1839,7 +1865,7 @@ dependencies = [ "mime", "native-tls", "percent-encoding 2.1.0", - "pin-project-lite 0.2.6", + "pin-project-lite", "serde", "serde_urlencoded 0.7.0", "tokio 1.3.0", @@ -1899,16 +1925,17 @@ dependencies = [ [[package]] name = "scootaloo" -version = "0.2.1" +version = "0.3.2" dependencies = [ "clap", "egg-mode", "elefren", "htmlescape", + "log", "reqwest 0.11.2", "serde", + "simple_logger", "tokio 1.3.0", - "tokio-compat-02", "toml", ] @@ -2061,6 +2088,28 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "simple_logger" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd57f17c093ead1d4a1499dc9acaafdd71240908d64775465543b8d9a9f1d198" +dependencies = [ + "atty", + "chrono", + "colored", + "log", + "winapi 0.3.8", +] + [[package]] name = "siphasher" version = "0.2.3" @@ -2257,24 +2306,6 @@ dependencies = [ "tokio-timer", ] -[[package]] -name = "tokio" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" -dependencies = [ - "bytes 0.5.6", - "fnv", - "iovec", - "lazy_static", - "memchr", - "mio 0.6.23", - "num_cpus", - "pin-project-lite 0.1.4", - "slab", - "tokio-macros", -] - [[package]] name = "tokio" version = "1.3.0" @@ -2287,7 +2318,12 @@ dependencies = [ "memchr", "mio 0.7.9", "num_cpus", - "pin-project-lite 0.2.6", + "once_cell", + "parking_lot 0.11.1", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi 0.3.8", ] [[package]] @@ -2301,20 +2337,6 @@ dependencies = [ "futures 0.1.29", ] -[[package]] -name = "tokio-compat-02" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d4237822b7be8fff0a7a27927462fad435dcb6650f95cea9e946bf6bdc7e07" -dependencies = [ - "bytes 0.5.6", - "once_cell", - "pin-project-lite 0.2.6", - "tokio 0.2.25", - "tokio 1.3.0", - "tokio-stream", -] - [[package]] name = "tokio-current-thread" version = "0.1.7" @@ -2348,9 +2370,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "0.2.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2", "quote", @@ -2379,24 +2401,13 @@ dependencies = [ "log", "mio 0.6.23", "num_cpus", - "parking_lot", + "parking_lot 0.9.0", "slab", "tokio-executor", "tokio-io", "tokio-sync", ] -[[package]] -name = "tokio-stream" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469" -dependencies = [ - "futures-core", - "pin-project-lite 0.2.6", - "tokio 1.3.0", -] - [[package]] name = "tokio-sync" version = "0.1.8" @@ -2450,30 +2461,6 @@ dependencies = [ "tokio-executor", ] -[[package]] -name = "tokio-tls" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828" -dependencies = [ - "native-tls", - "tokio 0.2.25", -] - -[[package]] -name = "tokio-util" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" -dependencies = [ - "bytes 0.5.6", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.1.4", - "tokio 0.2.25", -] - [[package]] name = "tokio-util" version = "0.6.4" @@ -2484,7 +2471,7 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.6", + "pin-project-lite", "tokio 1.3.0", ] @@ -2510,7 +2497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", - "pin-project-lite 0.2.6", + "pin-project-lite", "tracing-core", ] @@ -2703,6 +2690,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasm-bindgen" version = "0.2.71" diff --git a/Cargo.toml b/Cargo.toml index 02efb79..21850ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scootaloo" -version = "0.2.1" +version = "0.3.2" authors = ["VC "] edition = "2018" @@ -8,18 +8,12 @@ edition = "2018" [dependencies] serde = { version = "1.0", features = ["derive"] } - toml = "^0.5" - clap = "^2.33" - -tokio = { version = "1", features = ["rt-multi-thread"]} -tokio-compat-02 = "0.2" - -egg-mode = "^0.15" - +egg-mode = { git = "https://github.com/egg-mode-rs/egg-mode", rev = "6b81073eba9c3b123ca0e80bdb5ef61d1758f131" } elefren = "^0.22" - -reqwest = { version="^0.11", features = ["blocking"] } - +tokio = { version = "1", features = ["full"]} +reqwest = "^0.11" htmlescape = "^0.3" +log = "^0.4" +simple_logger = "^1.11" diff --git a/README.md b/README.md index 65c9986..ed5b972 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A Twitter to Mastodon copy bot written in Rust It: * copies the content (text) of the original Tweet * dereferences the links -* gets every attach media (photo, video or gif) +* gets every attached media (photo, video or gif) If any of the last steps failed, the Toot gets published with the exact same text as the Tweet. @@ -74,4 +74,3 @@ echo -n '8189881949849' > last_tweet **This file should only contain the last tweet ID without any other char (no EOL or new line).** -Oh and everything is sync (and not async) so this does not run at a blazing speed… diff --git a/src/lib.rs b/src/lib.rs index 2033a13..c7fadbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,18 +1,14 @@ // std use std::{ - path::Path, borrow::Cow, collections::HashMap, - io::{stdin, copy}, + io::stdin, fmt, - fs::{read_to_string, write, create_dir_all, File, remove_file}, + fs::{read_to_string, write}, error::Error, + sync::{Arc, Mutex}, }; -//tokio -use tokio::runtime::Runtime; -use tokio_compat_02::FutureExt; - // toml use serde::Deserialize; @@ -37,34 +33,45 @@ use elefren::{ }; // reqwest -use reqwest::blocking::Client; +use reqwest::Url; + +// tokio +use tokio::{ + io::copy, + fs::{File, create_dir_all, remove_file}, + sync::mpsc, +}; // htmlescape use htmlescape::decode_html; +// log +use log::{info, warn, error, debug}; + /********** * Generic usage functions ***********/ /* * Those functions are related to the Twitter side of things */ -/// Read last tweet id from a file +/// Reads last tweet id from a file fn read_state(s: &str) -> Option { let state = read_to_string(s); if let Ok(s) = state { + debug!("Last Tweet ID (from file): {}", &s); return s.parse::().ok(); } None } -/// Write last treated tweet id to a file +/// Writes last treated tweet id to a file fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> { write(f, format!("{}", s)) } -/// Get twitter oauth2 token +/// Gets Twitter oauth2 token fn get_oauth2_token(config: &Config) -> Token { let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret)); let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret)); @@ -75,19 +82,18 @@ fn get_oauth2_token(config: &Config) -> Token { } } -/// Get twitter user timeline -fn get_user_timeline(config: &Config, token: Token, lid: Option) -> Result, Box> { +/// Gets Twitter user timeline +async fn get_user_timeline(config: &Config, token: Token, lid: Option) -> Result, Box> { // fix the page size to 200 as it is the maximum Twitter authorizes - let rt = Runtime::new()?; - let (_timeline, feed) = rt.block_on(user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token) + let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token) .with_page_size(200) .older(lid) - .compat())?; + .await?; Ok(feed.to_vec()) } -/// decode urls from UrlEntities +/// Decodes urls from UrlEntities fn decode_urls(urls: &Vec) -> HashMap { let mut decoded_urls = HashMap::new(); @@ -101,6 +107,8 @@ fn decode_urls(urls: &Vec) -> HashMap { decoded_urls } +/// Decodes the Twitter mention to something that will make sense once Twitter has joined the +/// Fediverse fn twitter_mentions(ums: &Vec) -> HashMap { let mut decoded_mentions = HashMap::new(); @@ -111,18 +119,18 @@ fn twitter_mentions(ums: &Vec) -> HashMap { decoded_mentions } -/// Retrieve a single media from a tweet and store it in a temporary file -fn get_tweet_media(m: &MediaEntity, t: &str) -> Result> { +/// Retrieves a single media from a tweet and store it in a temporary file +async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result> { match m.media_type { MediaType::Photo => { - return cache_media(&m.media_url_https, t); + return cache_media(&m.media_url_https, t).await; }, _ => { match &m.video_info { Some(v) => { for variant in &v.variants { if variant.content_type == "video/mp4" { - return cache_media(&variant.url, t); + return cache_media(&variant.url, t).await; } } return Err(Box::new(ScootalooError::new(format!("Media Type for {} is video but no mp4 file URL is available", &m.url).as_str()))); @@ -138,7 +146,7 @@ fn get_tweet_media(m: &MediaEntity, t: &str) -> Result> { /* * Those functions are related to the Mastodon side of things */ -/// Get Mastodon Data +/// Gets Mastodon Data fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon { let data = Data { base: Cow::from(String::from(&masto.base)), @@ -151,7 +159,7 @@ fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon { Mastodon::from(data) } -/// build toot text from tweet +/// Builds toot text from tweet fn build_basic_status(tweet: &Tweet) -> Result> { let mut toot = String::from(&tweet.text); @@ -177,35 +185,40 @@ fn build_basic_status(tweet: &Tweet) -> Result> { /* * Generic private functions */ -fn cache_media(u: &str, t: &str) -> Result> { + +/// Gets and caches Twitter Media inside the determined temp dir +async fn cache_media(u: &str, t: &str) -> Result> { // create dir - if !Path::new(t).is_dir() { - create_dir_all(t)?; - } + create_dir_all(t).await?; // get file - let client = Client::new(); - let mut response = client.get(u).send()?; + let mut response = reqwest::get(u).await?; // create local file - let dest_filename = match response.url() - .path_segments() - .and_then(|segments| segments.last()) { - Some(r) => r, - None => { - return Err(Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str()))); - }, - }; + let url = Url::parse(u)?; + let dest_filename = url.path_segments().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))? + .last().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?; let dest_filepath = format!("{}/{}", t, dest_filename); - let mut dest_file = File::create(&dest_filepath)?; + let mut dest_file = File::create(&dest_filepath).await?; - copy(&mut response, &mut dest_file)?; + while let Some(chunk) = response.chunk().await? { + copy(&mut &*chunk, &mut dest_file).await?; + } Ok(dest_filepath) } +/********** + * This is the struct that holds the Mastodon Media ID and the Twitter Media URL at the same Time +**********/ +#[derive(Debug)] +struct ScootalooSpawnResponse { + mastodon_media_id: String, + twitter_media_url: String, +} + /********** * local error handler **********/ @@ -287,7 +300,7 @@ pub fn parse_toml(toml_file: &str) -> Config { /// Generic register function /// As this function is supposed to be run only once, it will panic for every error it encounters -/// Most of this function is a direct copy/paste of the official `mammut` crate +/// Most of this function is a direct copy/paste of the official `elefren` crate pub fn register(host: &str) { let mut builder = App::builder(); builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME")))) @@ -315,7 +328,8 @@ pub fn register(host: &str) { } /// This is where the magic happens -pub fn run(config: Config) { +#[tokio::main] +pub async fn run(config: Config) { // retrieve the last tweet ID for the username let last_tweet_id = read_state(&config.scootaloo.last_tweet_path); @@ -323,16 +337,18 @@ pub fn run(config: Config) { let token = get_oauth2_token(&config); // get Mastodon instance - let mastodon = get_mastodon_token(&config.mastodon); + let mastodon = Arc::new(Mutex::new(get_mastodon_token(&config.mastodon))); // get user timeline feed (Vec) - let mut feed = get_user_timeline(&config, token, last_tweet_id).unwrap_or_else(|e| + let mut feed = get_user_timeline(&config, token, last_tweet_id) + .await + .unwrap_or_else(|e| panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e) ); // empty feed -> exiting if feed.is_empty() { - println!("Nothing to retrieve since last time, exiting…"); + info!("Nothing to retrieve since last time, exiting…"); return; } @@ -340,10 +356,12 @@ pub fn run(config: Config) { feed.reverse(); for tweet in &feed { + debug!("Treating Tweet {} inside feed", tweet.id); // determine if the tweet is part of a thread (response to self) or a standard response if let Some(r) = &tweet.in_reply_to_screen_name { if &r.to_lowercase() != &config.twitter.username.to_lowercase() { // we are responding not threading + info!("Tweet is a direct response, skipping"); continue; } }; @@ -352,7 +370,7 @@ pub fn run(config: Config) { let mut status_text = match build_basic_status(tweet) { Ok(t) => t, Err(e) => { - println!("Could not create status from tweet {}: {}", tweet.id ,e); + error!("Could not create status from tweet {}: {}", tweet.id ,e); continue; }, }; @@ -361,35 +379,64 @@ pub fn run(config: Config) { // reupload the attachments if any if let Some(m) = &tweet.extended_entities { + let (tx, mut rx) = mpsc::channel(4); + for media in &m.media { - let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path) { - Ok(m) => m, - Err(e) => { - println!("Cannot get tweet media for {}: {}", &media.url, e); - continue; - }, - }; + // creating a new tx for this initial loop + let tx = tx.clone(); + // creating a new mastodon from the original mutex + let mastodon = mastodon.clone(); + // unfortunately for this to be thread safe, we need to clone a lot of structures + let media = media.clone(); + let cache_path = config.scootaloo.cache_path.clone(); - let mastodon_media_ids = match mastodon.media(Cow::from(String::from(&local_tweet_media_path))) { - Ok(m) => { - remove_file(&local_tweet_media_path).unwrap_or_else(|e| - println!("Attachment for {} has been upload, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e) - ); - m.id - }, - Err(e) => { - println!("Cannot attach media {} to Mastodon Instance: {}", &local_tweet_media_path, e); - continue; + tokio::spawn(async move { + debug!("Spawing new async thread to treat {}", &media.id); + let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await { + Ok(m) => m, + Err(e) => { + // we could have panicked here, no issue, but I’m not confortable using + // that for now + warn!("Cannot get tweet media for {}: {}", &media.url, e); + return; + } + }; + + // we cannot directly do all the stuff inside here because mastodon lock can + // live outside this + let mas_result = mastodon.lock().unwrap().media(Cow::from(String::from(&local_tweet_media_path))); + + match mas_result { + Ok(m) => { + remove_file(&local_tweet_media_path).await.unwrap_or_else(|e| + warn!("Attachment {} has been uploaded but I’m unable to remove the existing file: {}", &local_tweet_media_path, e) + ); + // we can unwrap here because we’re in a thread + tx.send(ScootalooSpawnResponse { + mastodon_media_id: m.id.clone(), + twitter_media_url: local_tweet_media_path.clone() + }).await.unwrap(); + }, + Err(e) => { + error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e); + } } - }; + }); + } - status_medias.push(mastodon_media_ids); + // dropping the last tx otherwise recv() will wait indefinitely + drop(tx); - // last step, removing the reference to the media from with the toot’s text - status_text = status_text.replace(&media.url, ""); + while let Some(i) = rx.recv().await { + // pushes the media into the media vec + status_medias.push(i.mastodon_media_id); + // removes the URL from the original Tweet text + status_text = status_text.replace(&i.twitter_media_url, ""); } } + // finished reuploading attachments, now let’s do the toot baby! + debug!("Building corresponding Mastodon status"); let status = StatusBuilder::new() .status(&status_text) .media_ids(status_medias) @@ -397,7 +444,8 @@ pub fn run(config: Config) { .expect(format!("Cannot build status with text {}", &status_text).as_str()); // publish status - mastodon.new_status(status).unwrap(); + // again unwrap is safe here as we are in the main thread + mastodon.lock().unwrap().new_status(status).unwrap(); // this will panic if it cannot publish the status, which is a good thing, it allows the // last_tweet gathered not to be written diff --git a/src/main.rs b/src/main.rs index 575358a..529faa7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,13 @@ use scootaloo::*; // clap use clap::{App, Arg, SubCommand}; +// log +use log::{LevelFilter, error}; +use simple_logger::SimpleLogger; + +// std +use std::str::FromStr; + fn main() { let matches = App::new(env!("CARGO_PKG_NAME")) .version(env!("CARGO_PKG_VERSION")) @@ -15,6 +22,13 @@ fn main() { .help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)") .takes_value(true) .display_order(1)) + .arg(Arg::with_name("log_level") + .short("l") + .long("loglevel") + .value_name("LOGLEVEL") + .help("Log level. Valid values are: Off, Warn, Error, Info, Debug") + .takes_value(true) + .display_order(2)) .subcommand(SubCommand::with_name("register") .version(env!("CARGO_PKG_VERSION")) .about("Command to register to a Mastodon Instance") @@ -32,7 +46,18 @@ fn main() { return; } + if matches.is_present("log_level") { + match LevelFilter::from_str(matches.value_of("log_level").unwrap()) { + Ok(level) => { SimpleLogger::new().with_level(level).init().unwrap()}, + Err(e) => { + SimpleLogger::new().with_level(LevelFilter::Error).init().unwrap(); + error!("Unknown log level filter: {}", e); + } + }; + } + let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml")); run(config); } +