Merge branch 'fix_smart_mentions' into 'master'

Fix smart mentions

See merge request veretcle/scootaloo!37

Cargo.lock

@@ -46,15 +46,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -261,9 +252,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
+checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
 dependencies = [
  "iana-time-zone",
  "js-sys",
@@ -277,17 +268,24 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "2.34.0"
+version = "4.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+checksum = "60494cedb60cb47462c0ff7be53de32c0e42a6fc2c772184554fa12bd9489c03"
 dependencies = [
- "ansi_term",
  "atty",
  "bitflags",
+ "clap_lex",
  "strsim",
- "textwrap",
+ "termcolor",
- "unicode-width",
+]
- "vec_map",
+
+[[package]]
+name = "clap_lex"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8"
+dependencies = [
+ "os_str_bytes",
 ]
 
 [[package]]
@@ -1567,6 +1565,12 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "os_str_bytes"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b5bf27447411e9ee3ff51186bf7a08e16c341efdde93f4d823e8844429bed7e"
+
 [[package]]
 name = "parking_lot"
 version = "0.9.0"
@@ -2099,7 +2103,7 @@ dependencies = [
 
 [[package]]
 name = "scootaloo"
-version = "0.8.2"
+version = "0.10.1"
 dependencies = [
  "chrono",
  "clap",
@@ -2368,9 +2372,9 @@ dependencies = [
 
 [[package]]
 name = "strsim"
-version = "0.8.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "subtle"
@@ -2430,15 +2434,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.37"
@@ -2869,12 +2864,6 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
 [[package]]
 name = "version_check"
 version = "0.9.4"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scootaloo"
-version = "0.8.2"
+version = "0.10.1"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
@@ -10,11 +10,11 @@ edition = "2021"
 chrono = "^0.4"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^2.34"
+clap = "^4"
-futures = "^0.3"
 egg-mode = "^0.16"
 rusqlite = "^0.27"
-tokio = { version = "1", features = ["full"]}
+tokio = { version = "^1", features = ["full"]}
+futures = "^0.3"
 elefren = "^0.22"
 html-escape = "^0.2"
 reqwest = "^0.11"

README.md

@@ -18,9 +18,11 @@ First up, create a configuration file (default path is `/usr/local/etc/scootaloo
 [scootaloo]
 db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
 cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
+rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneously
 
 [twitter]
 ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
+page_size = 20 ## optional, default 200, max number of tweet retrieved
 consumer_key = "MYCONSUMERKEY"
 consumer_secret = "MYCONSUMERSECRET"
 access_key = "MYACCESSKEY"
@@ -46,6 +48,7 @@ This will give you the end of the TOML file. It will look like this:
 ```toml
 [mastodon.nintendojofr] ## account
 twitter_screen_name = "NintendojoFR" ## User Timeline to copy
+mastodon_screen_name = "nintendojofr" ## optional, Mastodon account name used for smart mentions
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
 client_secret = "MYCLIENTSECRET"
@@ -55,6 +58,11 @@ token = "MYTOKEN"
 
 You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
 
+You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so:
+```
+twitter_page_size = 40
+```
+
 ## Running
 
 You can then run the application via `cron` for example. Here is the generic usage:

src/config.rs

@@ -16,11 +16,14 @@ pub struct TwitterConfig {
     pub consumer_secret: String,
     pub access_key: String,
     pub access_secret: String,
+    pub page_size: Option<i32>,
 }
 
 #[derive(Debug, Deserialize)]
 pub struct MastodonConfig {
     pub twitter_screen_name: String,
+    pub mastodon_screen_name: Option<String>,
+    pub twitter_page_size: Option<i32>,
     pub base: String,
     pub client_id: String,
     pub client_secret: String,
@@ -32,6 +35,7 @@ pub struct MastodonConfig {
 pub struct ScootalooConfig {
     pub db_path: String,
     pub cache_path: String,
+    pub rate_limit: Option<usize>,
 }
 
 /// Parses the TOML file into a Config Struct

src/lib.rs

@@ -19,18 +19,19 @@ mod state;
 pub use state::{init_db, migrate_db};
 use state::{read_state, write_state, TweetToToot};
 
-use elefren::{prelude::*, status_builder::StatusBuilder};
+use elefren::{prelude::*, status_builder::StatusBuilder, Language};
+use futures::StreamExt;
 use log::info;
 use rusqlite::Connection;
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 use tokio::{spawn, sync::Mutex};
 
+const DEFAULT_RATE_LIMIT: usize = 4;
+const DEFAULT_PAGE_SIZE: i32 = 200;
+
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
-    // create the task vector for handling multiple accounts
-    let mut mtask = vec![];
-
     // open the SQLite connection
     let conn = Arc::new(Mutex::new(
         Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
@@ -41,64 +42,73 @@ pub async fn run(config: Config) {
         }),
     ));
 
-    for mastodon_config in config.mastodon.into_values() {
+    let scootaloo_mentions: HashMap<String, String> = config
+        .mastodon
+        .values()
+        .filter(|s| s.mastodon_screen_name.is_some())
+        .map(|s| {
+            (
+                format!("@{}", s.twitter_screen_name),
+                format!(
+                    "@{}@{}",
+                    s.mastodon_screen_name.as_ref().unwrap(),
+                    s.base.split('/').last().unwrap()
+                ),
+            )
+        })
+        .collect();
+
+    let mut stream = futures::stream::iter(config.mastodon.into_values())
+        .map(|mastodon_config| {
+            // calculate Twitter page size
+            let page_size = mastodon_config
+                .twitter_page_size
+                .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
             // create temporary value for each task
             let scootaloo_cache_path = config.scootaloo.cache_path.clone();
+            let scootaloo_mentions = scootaloo_mentions.clone();
             let token = get_oauth2_token(&config.twitter);
             let task_conn = conn.clone();
 
-        let task = spawn(async move {
+            spawn(async move {
                 info!("Starting treating {}", &mastodon_config.twitter_screen_name);
-
                 // retrieve the last tweet ID for the username
                 let lconn = task_conn.lock().await;
-            let last_tweet_id =
+                let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
-                read_state(&lconn, &mastodon_config.twitter_screen_name, None)?.map(|r| r.tweet_id);
+                    .map(|r| r.tweet_id);
                 drop(lconn);
 
-            // get user timeline feed (Vec<tweet>)
+                // get reversed, curated user timeline
-            let mut feed =
+                let feed = get_user_timeline(
-                get_user_timeline(&mastodon_config.twitter_screen_name, &token, last_tweet_id)
+                    &mastodon_config.twitter_screen_name,
+                    &token,
+                    last_tweet_id,
+                    page_size,
+                )
                 .await?;
 
-            // empty feed -> exiting
-            if feed.is_empty() {
-                info!("Nothing to retrieve since last time, exiting…");
-                return Ok(());
-            }
-
                 // get Mastodon instance
                 let mastodon = get_mastodon_token(&mastodon_config);
 
-            // order needs to be chronological
-            feed.reverse();
-
                 for tweet in &feed {
                     info!("Treating Tweet {} inside feed", tweet.id);
-                // initiate the toot_reply_id var
+
-                let mut toot_reply_id: Option<String> = None;
-                // determine if the tweet is part of a thread (response to self) or a standard response
-                if let Some(r) = &tweet.in_reply_to_screen_name {
-                    if r.to_lowercase() != mastodon_config.twitter_screen_name.to_lowercase() {
-                        // we are responding not threading
-                        info!("Tweet is a direct response, skipping");
-                        continue;
-                    }
-                    info!("Tweet is a thread");
-                    // get the corresponding toot id
                     let lconn = task_conn.lock().await;
-                    toot_reply_id = read_state(
+                    // initiate the toot_reply_id var and retrieve the corresponding toot_id
+                    let toot_reply_id: Option<String> = tweet.in_reply_to_user_id.and_then(|_| {
+                        read_state(
                             &lconn,
                             &mastodon_config.twitter_screen_name,
                             tweet.in_reply_to_status_id,
                         )
                         .unwrap_or(None)
-                    .map(|s| s.toot_id);
+                        .map(|s| s.toot_id)
+                    });
                     drop(lconn);
-                };
 
                     // build basic status by just yielding text and dereferencing contained urls
-                let mut status_text = build_basic_status(tweet);
+                    let mut status_text = build_basic_status(tweet, &scootaloo_mentions);
 
                     // building associative media list
                     let (media_url, status_medias) =
@@ -112,10 +122,18 @@ pub async fn run(config: Config) {
 
                     status_builder.status(&status_text).media_ids(status_medias);
 
+                    // theard if necessary
                     if let Some(i) = toot_reply_id {
                         status_builder.in_reply_to(&i);
                     }
 
+                    // language if any
+                    if let Some(l) = &tweet.lang {
+                        if let Some(r) = Language::from_639_1(l) {
+                            status_builder.language(r);
+                        }
+                    }
+
                     // can be activated for test purposes
                     // status_builder.visibility(elefren::status_builder::Visibility::Private);
 
@@ -137,15 +155,13 @@ pub async fn run(config: Config) {
                     drop(lconn);
                 }
                 Ok::<(), ScootalooError>(())
-        });
+            })
-
+        })
-        // push each task into the vec task
+        .buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
-        mtask.push(task);
-    }
 
     // launch and wait for every handle
-    for handle in mtask {
+    while let Some(result) = stream.next().await {
-        match handle.await {
+        match result {
             Ok(Err(e)) => eprintln!("Error within thread: {}", e),
             Err(e) => eprintln!("Error with thread: {}", e),
             _ => (),

src/main.rs

@@ -1,5 +1,5 @@
-use clap::{App, Arg, SubCommand};
+use clap::{Arg, Command};
-use log::{error, LevelFilter};
+use log::LevelFilter;
 use scootaloo::*;
 use simple_logger::SimpleLogger;
 use std::str::FromStr;
@@ -7,115 +7,118 @@ use std::str::FromStr;
 const DEFAULT_CONFIG_PATH: &str = "/usr/local/etc/scootaloo.toml";
 
 fn main() {
-    let matches = App::new(env!("CARGO_PKG_NAME"))
+    let matches = Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))
         .about("A Twitter to Mastodon bot")
         .arg(
-            Arg::with_name("config")
+            Arg::new("config")
-                .short("c")
+                .short('c')
                 .long("config")
                 .value_name("CONFIG_FILE")
-                .help(&format!(
+                .help("TOML config file for scootaloo")
-                    "TOML config file for scootaloo (default {})",
+                .num_args(1)
-                    DEFAULT_CONFIG_PATH
+                .default_value(DEFAULT_CONFIG_PATH)
-                ))
-                .takes_value(true)
                 .display_order(1),
         )
         .arg(
-            Arg::with_name("log_level")
+            Arg::new("log_level")
-                .short("l")
+                .short('l')
                 .long("loglevel")
                 .value_name("LOGLEVEL")
-                .help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
+                .help("Log level")
-                .takes_value(true)
+                .num_args(1)
+                .value_parser(["Off", "Warn", "Error", "Info", "Debug"])
                 .display_order(2),
         )
         .subcommand(
-            SubCommand::with_name("register")
+            Command::new("register")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to register to a Mastodon Instance")
                 .arg(
-                    Arg::with_name("host")
+                    Arg::new("host")
-                        .short("H")
+                        .short('H')
                         .long("host")
                         .value_name("HOST")
                         .help("Base URL of the Mastodon instance to register to (no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .required(true)
                         .display_order(1)
                     )
                 .arg(
-                    Arg::with_name("name")
+                    Arg::new("name")
-                        .short("n")
+                        .short('n')
                         .long("name")
                         .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .required(true)
                         .display_order(2)
                 ),
         )
         .subcommand(
-            SubCommand::with_name("init")
+            Command::new("init")
                 .version(env!("CARGO_PKG_VERSION"))
                 .about("Command to init Scootaloo DB")
                 .arg(
-                    Arg::with_name("config")
+                    Arg::new("config")
-                        .short("c")
+                        .short('c')
                         .long("config")
                         .value_name("CONFIG_FILE")
                         .help(&format!(
                             "TOML config file for scootaloo (default {})",
                             DEFAULT_CONFIG_PATH
                         ))
-                        .takes_value(true)
+                        .default_value(DEFAULT_CONFIG_PATH)
+                        .num_args(1)
                         .display_order(1),
                 ),
         )
         .subcommand(
-            SubCommand::with_name("migrate")
+            Command::new("migrate")
             .version(env!("CARGO_PKG_VERSION"))
             .about("Command to migrate Scootaloo DB")
             .arg(
-                Arg::with_name("config")
+                Arg::new("config")
-                    .short("c")
+                    .short('c')
                     .long("config")
                     .value_name("CONFIG_FILE")
                     .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
-                    .takes_value(true)
+                    .default_value(DEFAULT_CONFIG_PATH)
+                    .num_args(1)
                     .display_order(1),
             )
             .arg(
-                    Arg::with_name("name")
+                    Arg::new("name")
-                        .short("n")
+                        .short('n')
                         .long("name")
                         .help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
-                        .takes_value(true)
+                        .num_args(1)
                         .display_order(2)
             )
         )
         .get_matches();
 
     match matches.subcommand() {
-        ("register", Some(sub_m)) => {
+        Some(("register", sub_m)) => {
             register(
-                sub_m.value_of("host").unwrap(),
+                sub_m.get_one::<String>("host").unwrap(),
-                sub_m.value_of("name").unwrap(),
+                sub_m.get_one::<String>("name").unwrap(),
             );
             return;
         }
-        ("init", Some(sub_m)) => {
+        Some(("init", sub_m)) => {
-            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
             init_db(&config.scootaloo.db_path).unwrap();
             return;
         }
-        ("migrate", Some(sub_m)) => {
+        Some(("migrate", sub_m)) => {
-            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+            let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
             let config_twitter_screen_name =
                 &config.mastodon.values().next().unwrap().twitter_screen_name;
             migrate_db(
                 &config.scootaloo.db_path,
-                sub_m.value_of("name").unwrap_or(config_twitter_screen_name),
+                sub_m
+                    .get_one::<String>("name")
+                    .unwrap_or(config_twitter_screen_name),
             )
             .unwrap();
             return;
@@ -123,20 +126,14 @@ fn main() {
         _ => (),
     }
 
-    if matches.is_present("log_level") {
+    if let Some(level) = matches.get_one::<String>("log_level") {
-        match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
-            Ok(level) => SimpleLogger::new().with_level(level).init().unwrap(),
-            Err(e) => {
         SimpleLogger::new()
-                    .with_level(LevelFilter::Error)
+            .with_level(LevelFilter::from_str(level).unwrap())
             .init()
             .unwrap();
-                error!("Unknown log level filter: {}", e);
-            }
-        };
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+    let config = parse_toml(matches.get_one::<String>("config").unwrap());
 
     run(config);
 }

src/mastodon.rs

@@ -4,7 +4,7 @@ use egg_mode::{
     entities::{MentionEntity, UrlEntity},
     tweet::Tweet,
 };
-use elefren::{apps::App, prelude::*, scopes::Scopes};
+use elefren::{apps::App, prelude::*, scopes::Read, scopes::Scopes, scopes::Write};
 use html_escape::decode_html_entities;
 use std::{borrow::Cow, collections::HashMap, io::stdin};
 
@@ -48,14 +48,18 @@ pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
 }
 
 /// Builds toot text from tweet
-pub fn build_basic_status(tweet: &Tweet) -> String {
+pub fn build_basic_status(tweet: &Tweet, mentions: &HashMap<String, String>) -> String {
     let mut toot = tweet.text.to_owned();
 
     for decoded_url in decode_urls(&tweet.entities.urls) {
         toot = toot.replace(&decoded_url.0, &decoded_url.1);
     }
 
-    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions) {
+    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions)
+        .into_iter()
+        .chain(mentions.to_owned())
+        .collect::<HashMap<String, String>>()
+    {
         toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
     }
 
@@ -70,7 +74,12 @@ pub fn register(host: &str, screen_name: &str) {
     builder
         .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
         .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
-        .scopes(Scopes::write_all())
+        .scopes(
+            Scopes::write(Write::Accounts)
+                .and(Scopes::write(Write::Media))
+                .and(Scopes::write(Write::Statuses))
+                .and(Scopes::read(Read::Accounts)),
+        )
         .website(Cow::from(
             "https://framagit.org/veretcle/scootaloo".to_string(),
         ));
@@ -99,13 +108,19 @@ pub fn register(host: &str, screen_name: &str) {
 
     let toml = toml::to_string(&*mastodon).unwrap();
 
+    let current_account = mastodon
+        .verify_credentials()
+        .expect("Unable to access account information!");
+
     println!(
         "Please insert the following block at the end of your configuration file:
 [mastodon.{}]
 twitter_screen_name = \"{}\"
+mastodon_screen_name = \"{}\"
 {}",
         screen_name.to_lowercase(),
         screen_name,
+        current_account.username,
         toml
     );
 }
@@ -192,7 +207,13 @@ mod tests {
                             range: (80, 95),
                             name: "Nintendo France".to_string(),
                             screen_name: "NintendoFrance".to_string(),
-                        }
+                        },
+                        MentionEntity {
+                            id: 999999999,
+                            range: (80, 95),
+                            name: "Willy Wonka".to_string(),
+                            screen_name: "WillyWonka".to_string(),
+                        },
                     ],
                 media: None,
             },
@@ -213,7 +234,7 @@ mod tests {
             retweeted: None,
             retweeted_status: None,
             source: None,
-            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance https://t.co/zXw0FfX2Nt".to_string(),
+            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance @WillyWonka https://t.co/zXw0FfX2Nt".to_string(),
             truncated: false,
             user: None,
             withheld_copyright: false,
@@ -221,8 +242,13 @@ mod tests {
             withheld_scope: None,
         };
 
-        let t_out = build_basic_status(&t);
+        let s: HashMap<String, String> = HashMap::from([(
+            "@WillyWonka".to_string(),
+            "@WillyWonka@chocolatefactory.org".to_string(),
+        )]);
 
-        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com https://www.youtube.com/watch?v=w5TrSaoYmZ8");
+        let t_out = build_basic_status(&t, &s);
+
+        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com @WillyWonka@chocolatefactory.org https://www.youtube.com/watch?v=w5TrSaoYmZ8");
     }
 }

src/twitter.rs

@@ -27,19 +27,31 @@ pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
     }
 }
 
-/// Gets Twitter user timeline
+/// Gets Twitter user timeline, eliminate responses to others and reverse it
 pub async fn get_user_timeline(
     screen_name: &str,
     token: &Token,
     lid: Option<u64>,
+    page_size: i32,
 ) -> Result<Vec<Tweet>, Box<dyn Error>> {
     // fix the page size to 200 as it is the maximum Twitter authorizes
     let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
-        .with_page_size(200)
+        .with_page_size(page_size)
         .older(lid)
         .await?;
 
-    Ok(feed.to_vec())
+    let mut feed: Vec<Tweet> = feed
+        .iter()
+        .cloned()
+        .filter(|t| match &t.in_reply_to_screen_name {
+            Some(r) => r.to_lowercase() == screen_name.to_lowercase(),
+            None => true,
+        })
+        .collect();
+
+    feed.reverse();
+
+    Ok(feed)
 }
 
 /// Retrieves a single media from a tweet and store it in a temporary file

src/util.rs

@@ -15,6 +15,8 @@ use tokio::{
     io::copy,
 };
 
+use futures::{stream, stream::StreamExt};
+
 /// Generate associative table between media ids and tweet extended entities
 pub async fn generate_media_ids(
     tweet: &Tweet,
@@ -25,24 +27,20 @@ pub async fn generate_media_ids(
     let mut media_ids: Vec<String> = vec![];
 
     if let Some(m) = &tweet.extended_entities {
-        // create tasks list
-        let mut tasks = vec![];
-
-        // size of media_ids vector, should be equal to the media vector
-        media_ids.resize(m.media.len(), String::new());
-
         info!("{} medias in tweet", m.media.len());
 
-        for (i, media) in m.media.iter().enumerate() {
+        let medias = m.media.clone();
+
+        let mut stream = stream::iter(medias)
+            .map(|media| {
                 // attribute media url
                 media_url = media.url.clone();
 
                 // clone everything we need
                 let cache_path = String::from(cache_path);
-            let media = media.clone();
                 let mastodon = mastodon.clone();
 
-            let task = tokio::task::spawn(async move {
+                tokio::task::spawn(async move {
                     info!("Start treating {}", media.media_url_https);
                     // get the tweet embedded media
                     let local_tweet_media_path = get_tweet_media(&media, &cache_path).await?;
@@ -54,16 +52,15 @@ pub async fn generate_media_ids(
                     // it doesn’t matter if we can’t remove, cache_media fn is idempotent
                     remove_file(&local_tweet_media_path).await.ok();
 
-                Ok::<(usize, String), ScootalooError>((i, mastodon_media.id))
+                    Ok::<String, ScootalooError>(mastodon_media.id)
-            });
+                })
+            })
+            .buffered(4); // there are max four medias per tweet and they need to be treated in
+                          // order
 
-            tasks.push(task);
+        while let Some(result) = stream.next().await {
-        }
+            match result {
-
+                Ok(Ok(v)) => media_ids.push(v),
-        for task in tasks {
-            match task.await {
-                // insert the media at the right place
-                Ok(Ok((i, v))) => media_ids[i] = v,
                 Ok(Err(e)) => warn!("Cannot treat media: {}", e),
                 Err(e) => error!("Something went wrong when joining the main thread: {}", e),
             }

tests/config.rs

@@ -1,5 +1,80 @@
 use scootaloo::parse_toml;
 
+#[test]
+fn test_page_size() {
+    const DEFAULT_PAGE_SIZE: i32 = 200;
+    let toml = parse_toml("tests/page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, Some(100));
+
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42));
+
+    // this is the exact line that is used inside fn run() to determine the twitter page size
+    // passed to fn get_user_timeline()
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+    let page_size_for_1 = toml
+        .mastodon
+        .get("1")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 100);
+    assert_eq!(page_size_for_1, 42);
+
+    let toml = parse_toml("tests/no_page_size.toml");
+
+    assert_eq!(toml.twitter.page_size, None);
+    assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None);
+
+    // and same here
+    let page_size_for_0 = toml
+        .mastodon
+        .get("0")
+        .unwrap()
+        .twitter_page_size
+        .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
+
+    assert_eq!(page_size_for_0, 200);
+}
+
+#[test]
+fn test_parse_good_toml_rate_limit() {
+    let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml");
+
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize));
+}
+
+#[test]
+fn test_parse_good_toml_mastodon_screen_name() {
+    let parse_good_toml = parse_toml("tests/good_test_mastodon_screen_name.toml");
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("0")
+            .unwrap()
+            .mastodon_screen_name,
+        Some("tarace".to_string())
+    );
+
+    assert_eq!(
+        parse_good_toml
+            .mastodon
+            .get("1")
+            .unwrap()
+            .mastodon_screen_name,
+        None
+    );
+}
+
 #[test]
 fn test_parse_good_toml() {
     let parse_good_toml = parse_toml("tests/good_test.toml");
@@ -9,6 +84,7 @@ fn test_parse_good_toml() {
         "/var/random/scootaloo.sqlite"
     );
     assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
+    assert_eq!(parse_good_toml.scootaloo.rate_limit, None);
 
     assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
     assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");

tests/good_test_mastodon_screen_name.toml

@@ -0,0 +1,28 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+mastodon_screen_name="tarace"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/good_test_rate_limit.toml

@@ -0,0 +1,20 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+rate_limit=69
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.tamerelol]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/no_page_size.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"

tests/page_size.toml

@@ -0,0 +1,29 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+page_size=100
+
+[mastodon]
+[mastodon.0]
+twitter_screen_name="tamerelol"
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"
+
+[mastodon.1]
+twitter_screen_name="tonperemdr"
+twitter_page_size=42
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"