diff --git a/Cargo.lock b/Cargo.lock index 3c7b354..6849703 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2103,7 +2103,7 @@ dependencies = [ [[package]] name = "scootaloo" -version = "0.9.0" +version = "0.9.1" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 47873b6..a1305b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scootaloo" -version = "0.9.0" +version = "0.9.1" authors = ["VC "] edition = "2021" diff --git a/README.md b/README.md index be90edc..89fa0df 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ rate_limiting = 4 ## optional, default 4, number of accounts handled simultaneou [twitter] ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps) +page_size = 20 ## optional, default 200, max number of tweet retrieved consumer_key = "MYCONSUMERKEY" consumer_secret = "MYCONSUMERSECRET" access_key = "MYACCESSKEY" @@ -56,6 +57,11 @@ token = "MYTOKEN" You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts. +You can also add a custom twitter page size in this section that would override the global (under the `twitter` moniker) and default one (200), like so: +``` +twitter_page_size = 40 +``` + ## Running You can then run the application via `cron` for example. Here is the generic usage: diff --git a/src/config.rs b/src/config.rs index 35fa6f3..69d9219 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,11 +16,13 @@ pub struct TwitterConfig { pub consumer_secret: String, pub access_key: String, pub access_secret: String, + pub page_size: Option, } #[derive(Debug, Deserialize)] pub struct MastodonConfig { pub twitter_screen_name: String, + pub twitter_page_size: Option, pub base: String, pub client_id: String, pub client_secret: String, diff --git a/src/lib.rs b/src/lib.rs index 38fe096..2b2a18e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ use tokio::{spawn, sync::Mutex}; use futures::StreamExt; const DEFAULT_RATE_LIMIT: usize = 4; +const DEFAULT_PAGE_SIZE: i32 = 200; /// This is where the magic happens #[tokio::main] @@ -44,6 +45,11 @@ pub async fn run(config: Config) { let mut stream = futures::stream::iter(config.mastodon.into_values()) .map(|mastodon_config| { + // calculate Twitter page size + let page_size = mastodon_config + .twitter_page_size + .unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE)); + // create temporary value for each task let scootaloo_cache_path = config.scootaloo.cache_path.clone(); let token = get_oauth2_token(&config.twitter); @@ -59,9 +65,13 @@ pub async fn run(config: Config) { drop(lconn); // get user timeline feed (Vec) - let mut feed = - get_user_timeline(&mastodon_config.twitter_screen_name, &token, last_tweet_id) - .await?; + let mut feed = get_user_timeline( + &mastodon_config.twitter_screen_name, + &token, + last_tweet_id, + page_size, + ) + .await?; // empty feed -> exiting if feed.is_empty() { diff --git a/src/twitter.rs b/src/twitter.rs index 50e4ac8..01b2ad0 100644 --- a/src/twitter.rs +++ b/src/twitter.rs @@ -32,10 +32,11 @@ pub async fn get_user_timeline( screen_name: &str, token: &Token, lid: Option, + page_size: i32, ) -> Result, Box> { // fix the page size to 200 as it is the maximum Twitter authorizes let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token) - .with_page_size(200) + .with_page_size(page_size) .older(lid) .await?; diff --git a/tests/config.rs b/tests/config.rs index 3919033..ce35afc 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -1,5 +1,57 @@ use scootaloo::parse_toml; +#[test] +fn test_page_size() { + const DEFAULT_PAGE_SIZE: i32 = 200; + let toml = parse_toml("tests/page_size.toml"); + + assert_eq!(toml.twitter.page_size, Some(100)); + + assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None); + + assert_eq!(toml.mastodon.get("1").unwrap().twitter_page_size, Some(42)); + + // this is the exact line that is used inside fn run() to determine the twitter page size + // passed to fn get_user_timeline() + let page_size_for_0 = toml + .mastodon + .get("0") + .unwrap() + .twitter_page_size + .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE)); + let page_size_for_1 = toml + .mastodon + .get("1") + .unwrap() + .twitter_page_size + .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE)); + + assert_eq!(page_size_for_0, 100); + assert_eq!(page_size_for_1, 42); + + let toml = parse_toml("tests/no_page_size.toml"); + + assert_eq!(toml.twitter.page_size, None); + assert_eq!(toml.mastodon.get("0").unwrap().twitter_page_size, None); + + // and same here + let page_size_for_0 = toml + .mastodon + .get("0") + .unwrap() + .twitter_page_size + .unwrap_or_else(|| toml.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE)); + + assert_eq!(page_size_for_0, 200); +} + +#[test] +fn test_parse_good_toml_rate_limit() { + let parse_good_toml = parse_toml("tests/good_test_rate_limit.toml"); + + assert_eq!(parse_good_toml.scootaloo.rate_limit, Some(69 as usize)); +} + #[test] fn test_parse_good_toml() { let parse_good_toml = parse_toml("tests/good_test.toml"); @@ -9,6 +61,7 @@ fn test_parse_good_toml() { "/var/random/scootaloo.sqlite" ); assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo"); + assert_eq!(parse_good_toml.scootaloo.rate_limit, None); assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key"); assert_eq!(parse_good_toml.twitter.consumer_secret, "secret"); diff --git a/tests/good_test_rate_limit.toml b/tests/good_test_rate_limit.toml new file mode 100644 index 0000000..27b84ea --- /dev/null +++ b/tests/good_test_rate_limit.toml @@ -0,0 +1,20 @@ +[scootaloo] + +db_path="/var/random/scootaloo.sqlite" +cache_path="/tmp/scootaloo" +rate_limit=69 + +[twitter] +consumer_key="rand consumer key" +consumer_secret="secret" +access_key="rand access key" +access_secret="super secret" + +[mastodon] +[mastodon.tamerelol] +twitter_screen_name="tamerelol" +base = "https://m.nintendojo.fr" +client_id = "rand client id" +client_secret = "secret" +redirect = "urn:ietf:wg:oauth:2.0:oob" +token = "super secret" diff --git a/tests/no_page_size.toml b/tests/no_page_size.toml new file mode 100644 index 0000000..cfe544f --- /dev/null +++ b/tests/no_page_size.toml @@ -0,0 +1,19 @@ +[scootaloo] + +db_path="/var/random/scootaloo.sqlite" +cache_path="/tmp/scootaloo" + +[twitter] +consumer_key="rand consumer key" +consumer_secret="secret" +access_key="rand access key" +access_secret="super secret" + +[mastodon] +[mastodon.0] +twitter_screen_name="tamerelol" +base = "https://m.nintendojo.fr" +client_id = "rand client id" +client_secret = "secret" +redirect = "urn:ietf:wg:oauth:2.0:oob" +token = "super secret" diff --git a/tests/page_size.toml b/tests/page_size.toml new file mode 100644 index 0000000..d68df0d --- /dev/null +++ b/tests/page_size.toml @@ -0,0 +1,29 @@ +[scootaloo] + +db_path="/var/random/scootaloo.sqlite" +cache_path="/tmp/scootaloo" + +[twitter] +consumer_key="rand consumer key" +consumer_secret="secret" +access_key="rand access key" +access_secret="super secret" +page_size=100 + +[mastodon] +[mastodon.0] +twitter_screen_name="tamerelol" +base = "https://m.nintendojo.fr" +client_id = "rand client id" +client_secret = "secret" +redirect = "urn:ietf:wg:oauth:2.0:oob" +token = "super secret" + +[mastodon.1] +twitter_screen_name="tonperemdr" +twitter_page_size=42 +base = "https://m.nintendojo.fr" +client_id = "rand client id" +client_secret = "secret" +redirect = "urn:ietf:wg:oauth:2.0:oob" +token = "super secret"