Merge branch 'useless_crate' into 'master'

refactor: remove useless crate:: ref

See merge request veretcle/scootaloo!18

.gitlab-ci.yml

@@ -1,13 +1,15 @@
+---
+
 stages:
-      - build
+  - build
 
 rust-latest:
-    stage: build
+  stage: build
-    artifacts:
+  artifacts:
-      paths:
+    paths:
-        - target/release/scootaloo
+      - target/release/scootaloo
-    image: rust:latest
+  image: rust:latest
-    script:
+  script:
-        - cargo build --release --verbose
+    - cargo test
-        - strip target/release/${CI_PROJECT_NAME}
+    - cargo build --release --verbose
-
+    - strip target/release/${CI_PROJECT_NAME}

Cargo.toml

@@ -1,26 +1,23 @@
 [package]
 name = "scootaloo"
-version = "0.3.3"
+version = "0.5.1"
 authors = ["VC <veretcle+framagit@mateu.be>"]
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+chrono = "^0.4"
 serde = { version = "1.0", features = ["derive"] }
 toml = "^0.5"
-clap = "^2.33"
+clap = "^2.34"
-egg-mode = { git = "https://github.com/egg-mode-rs/egg-mode", rev = "6b81073eba9c3b123ca0e80bdb5ef61d1758f131" }
+futures = "^0.3"
-elefren = "^0.22"
+egg-mode = "^0.16"
+rusqlite = "^0.27"
 tokio = { version = "1", features = ["full"]}
+elefren = "^0.22"
+html-escape = "^0.2"
 reqwest = "^0.11"
-htmlescape = "^0.3"
 log = "^0.4"
-simple_logger = "^1.11"
+simple_logger = "^2.1"
-
-[profile.release]
-opt-level = 's'  # Optimize for size.
-lto = true # Link Time Optimization (LTO)
-codegen-units = 1 # Set this to 1 to allow for maximum size reduction optimizations:
-panic = 'abort' # removes the need for this extra unwinding code.
 

README.md

@@ -7,16 +7,16 @@ It:
 
 If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
 
-RT are excluded, replies are included.but only the source threads are copied, not the actual replies to other Twitter users.
+RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
 
 # Usage
 
 First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this:
 
-```
+```toml
 [scootaloo]
 
-last_tweet_path="/usr/local/etc/last_tweet" ## file containing the last tweet id received, must be writable
+db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
 cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
 
 [twitter]
@@ -29,14 +29,21 @@ access_key="MYACCESSKEY"
 access_secret="MYACCESSSECRET"
 ```
 
-Then run the command with the `register` subcommand:
+Then run the command with the `init` subcommand to initiate the DB:
+```sh
+scootaloo init
 ```
+
+This subcommand is completely idempotent.
+
+Then run the command with the `register` subcommand:
+```sh
 scootaloo register --host https://m.nintendojo.fr
 ```
 
 This will give you the end of the TOML file. It will look like this:
 
-```
+```toml
 [mastodon]
 base = "https://m.nintendojo.fr"
 client_id = "MYCLIENTID"
@@ -47,7 +54,9 @@ token = "MYTOKEN"
 
 You can then run the application via `cron` for example. Here is the generic usage:
 
-```
+```sh
+A Twitter to Mastodon bot
+
 USAGE:
     scootaloo [OPTIONS] [SUBCOMMAND]
 
@@ -57,20 +66,25 @@ FLAGS:
 
 OPTIONS:
     -c, --config <CONFIG_FILE>    TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)
+    -l, --loglevel <LOGLEVEL>     Log level. Valid values are: Off, Warn, Error, Info, Debug
 
 SUBCOMMANDS:
     help        Prints this message or the help of the given subcommand(s)
+    init        Command to init Scootaloo DB
     register    Command to register to a Mastodon Instance
 ```
 
 # Quirks
 
-Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the `last_tweet` file before copying an old account.
+Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the DB file before copying an old account.
 
-You can do that with a command like:
+You can insert that Tweet number, by connecting to the DB you created:
-```
+```sh
-echo -n '8189881949849' > last_tweet
+sqlite3 /var/lib/scootaloo/scootaloo.sqlite
 ```
 
-**This file should only contain the last tweet ID without any other char (no EOL or new line).**
+And inserting the data:
 
+```sql
+INSERT INTO tweet_to_toot VALUES (1383782580412030982, "");
+```

src/config.rs

@@ -0,0 +1,48 @@
+use std::fs::read_to_string;
+use serde::Deserialize;
+
+/// General configuration Struct
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub twitter: TwitterConfig,
+    pub mastodon: MastodonConfig,
+    pub scootaloo: ScootalooConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TwitterConfig {
+    pub username: String,
+    pub consumer_key: String,
+    pub consumer_secret: String,
+    pub access_key: String,
+    pub access_secret: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct MastodonConfig {
+    pub base: String,
+    pub client_id: String,
+    pub client_secret: String,
+    pub redirect: String,
+    pub token: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ScootalooConfig {
+    pub db_path: String,
+    pub cache_path: String,
+}
+
+/// Parses the TOML file into a Config Struct
+pub fn parse_toml(toml_file: &str) -> Config {
+    let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
+        panic!("Cannot open config file {}: {}", toml_file, e)
+    );
+
+    let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
+        panic!("Cannot parse TOML file {}: {}", toml_file, e)
+    );
+
+    config
+}
+

src/error.rs

@@ -0,0 +1,27 @@
+use std::fmt;
+
+#[derive(Debug)]
+pub struct ScootalooError {
+    details: String,
+}
+
+impl ScootalooError {
+    pub fn new(msg: &str) -> ScootalooError {
+        ScootalooError {
+            details: msg.to_string(),
+        }
+    }
+}
+
+impl fmt::Display for ScootalooError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.details)
+    }
+}
+
+impl std::error::Error for ScootalooError {
+    fn description(&self) -> &str {
+        &self.details
+    }
+}
+

src/lib.rs

@@ -1,346 +1,55 @@
-// std
+mod error;
-use std::{
+use error::ScootalooError;
-    borrow::Cow,
-    collections::HashMap,
-    io::stdin,
-    fmt,
-    fs::{read_to_string, write},
-    error::Error,
-    sync::{Arc, Mutex},
-};
 
-// toml
+mod config;
-use serde::Deserialize;
+use config::Config;
+pub use config::parse_toml;
 
-// egg-mode
+mod mastodon;
-use egg_mode::{
+use mastodon::{get_mastodon_token, build_basic_status};
-    Token,
+pub use mastodon::register;
-    KeyPair,
-    entities::{UrlEntity, MediaEntity, MentionEntity, MediaType},
-    user::UserID,
-    tweet::{
-        Tweet,
-        user_timeline,
-    },
-};
 
-// elefren
+mod twitter;
+use twitter::*;
+
+mod util;
+
+mod state;
+use state::{read_state, write_state, TweetToToot};
+pub use state::init_db;
+
+use std::borrow::Cow;
+use tokio::fs::remove_file;
 use elefren::{
     prelude::*,
-    apps::App,
     status_builder::StatusBuilder,
-    scopes::Scopes,
 };
-
-// reqwest
-use reqwest::Url;
-
-// tokio
-use tokio::{
-    io::copy,
-    fs::{File, create_dir_all, remove_file},
-    sync::mpsc,
-};
-
-// htmlescape
-use htmlescape::decode_html;
-
-// log
 use log::{info, warn, error, debug};
-
+use rusqlite::Connection;
-/**********
- * Generic usage functions
-***********/
-/*
- * Those functions are related to the Twitter side of things
- */
-/// Reads last tweet id from a file
-fn read_state(s: &str) -> Option<u64> {
-    let state = read_to_string(s);
-
-    if let Ok(s) = state {
-        debug!("Last Tweet ID (from file): {}", &s);
-        return s.parse::<u64>().ok();
-    }
-
-    None
-}
-
-/// Writes last treated tweet id to a file
-fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
-    write(f, format!("{}", s))
-}
-
-/// Gets Twitter oauth2 token
-fn get_oauth2_token(config: &Config) -> Token {
-    let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
-    let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
-
-    Token::Access {
-        consumer: con_token,
-        access: access_token,
-    }
-}
-
-/// Gets Twitter user timeline
-async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
-    // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
-                .with_page_size(200)
-                .older(lid)
-                .await?;
-
-    Ok(feed.to_vec())
-}
-
-/// Decodes urls from UrlEntities
-fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
-    let mut decoded_urls = HashMap::new();
-
-    for url in urls {
-        if url.expanded_url.is_some() {
-            // unwrap is safe here as we just verified that there is something inside expanded_url
-            decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
-        }
-    }
-
-    decoded_urls
-}
-
-/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse
-fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
-    let mut decoded_mentions = HashMap::new();
-
-    for um in ums {
-        decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
-    }
-
-    decoded_mentions
-}
-
-/// Retrieves a single media from a tweet and store it in a temporary file
-async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
-    match m.media_type {
-        MediaType::Photo => {
-            return cache_media(&m.media_url_https, t).await;
-        },
-        _ => {
-            match &m.video_info {
-                Some(v) => {
-                    for variant in &v.variants {
-                        if variant.content_type == "video/mp4" {
-                            return cache_media(&variant.url, t).await;
-                        }
-                    }
-                    return Err(Box::new(ScootalooError::new(format!("Media Type for {} is video but no mp4 file URL is available", &m.url).as_str())));
-                },
-                None => {
-                    return Err(Box::new(ScootalooError::new(format!("Media Type for {} is video but does not contain any video_info", &m.url).as_str())));
-                },
-            }
-        },
-    };
-}
-
-/*
- * Those functions are related to the Mastodon side of things
- */
-/// Gets Mastodon Data
-fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
-        base: Cow::from(String::from(&masto.base)),
-        client_id: Cow::from(String::from(&masto.client_id)),
-        client_secret: Cow::from(String::from(&masto.client_secret)),
-        redirect: Cow::from(String::from(&masto.redirect)),
-        token: Cow::from(String::from(&masto.token)),
-    };
-
-    Mastodon::from(data)
-}
-
-/// Builds toot text from tweet
-fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
-    let mut toot = String::from(&tweet.text);
-
-    let decoded_urls = decode_urls(&tweet.entities.urls);
-
-    for decoded_url in decoded_urls {
-        toot = toot.replace(&decoded_url.0, &decoded_url.1);
-    }
-
-    let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
-
-    for decoded_mention in decoded_mentions {
-        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
-    }
-
-    if let Ok(t) = decode_html(&toot) {
-        toot = t;
-    }
-
-    Ok(toot)
-}
-
-/*
- * Generic private functions
- */
-
-/// Gets and caches Twitter Media inside the determined temp dir
-async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
-    // create dir
-    create_dir_all(t).await?;
-
-    // get file
-    let mut response = reqwest::get(u).await?;
-
-    // create local file
-    let url = Url::parse(u)?;
-    let dest_filename = url.path_segments().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?
-                    .last().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?;
-
-    let dest_filepath = format!("{}/{}", t, dest_filename);
-
-    let mut dest_file = File::create(&dest_filepath).await?;
-
-    while let Some(chunk) = response.chunk().await? {
-        copy(&mut &*chunk, &mut dest_file).await?;
-    }
-
-    Ok(dest_filepath)
-}
-
-/**********
- * This is the struct that holds the Mastodon Media ID and the Twitter Media URL at the same Time
-**********/
-#[derive(Debug)]
-struct ScootalooSpawnResponse {
-    mastodon_media_id: String,
-    twitter_media_url: String,
-}
-
-/**********
- * local error handler
-**********/
-#[derive(Debug)]
-struct ScootalooError {
-    details: String,
-}
-
-impl ScootalooError {
-    fn new(msg: &str) -> ScootalooError {
-        ScootalooError {
-            details: String::from(msg),
-        }
-    }
-}
-
-impl fmt::Display for ScootalooError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl std::error::Error for ScootalooError {
-    fn description(&self) -> &str {
-        &self.details
-    }
-}
-
-/**********
- * Config structure
-***********/
-/// General configuration Struct
-#[derive(Debug, Deserialize)]
-pub struct Config {
-    twitter: TwitterConfig,
-    mastodon: MastodonConfig,
-    scootaloo: ScootalooConfig,
-}
-
-#[derive(Debug, Deserialize)]
-struct TwitterConfig {
-    username: String,
-    consumer_key: String,
-    consumer_secret: String,
-    access_key: String,
-    access_secret: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct MastodonConfig {
-    base: String,
-    client_id: String,
-    client_secret: String,
-    redirect: String,
-    token: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct ScootalooConfig {
-    last_tweet_path: String,
-    cache_path: String,
-}
-
-/*********
- * Main functions
-*********/
-/// Parses the TOML file into a Config Struct
-pub fn parse_toml(toml_file: &str) -> Config {
-    let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
-        panic!("Cannot open config file {}: {}", toml_file, e)
-    );
-
-    let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
-        panic!("Cannot parse TOML file {}: {}", toml_file, e)
-    );
-
-    config
-}
-
-/// Generic register function
-/// As this function is supposed to be run only once, it will panic for every error it encounters
-/// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str) {
-    let mut builder = App::builder();
-    builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
-        .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
-        .scopes(Scopes::write_all())
-        .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
-
-    let app = builder.build().expect("Cannot build the app");
-
-    let registration = Registration::new(host).register(app).expect("Cannot build registration object");
-    let url = registration.authorize_url().expect("Cannot generate registration URI!");
-
-    println!("Click this link to authorize on Mastodon: {}", url);
-    println!("Paste the returned authorization code: ");
-
-    let mut input = String::new();
-    stdin().read_line(&mut input).expect("Unable to read back registration code!");
-
-    let code = input.trim();
-    let mastodon = registration.complete(code).expect("Unable to create access token!");
-
-    let toml = toml::to_string(&*mastodon).unwrap();
-
-    println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
-}
 
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
+    // open the SQLite connection
+    let conn = Connection::open(&config.scootaloo.db_path)
+        .unwrap_or_else(|e|
+            panic!("Something went wrong when opening the DB {}: {}", &config.scootaloo.db_path, e)
+        );
     // retrieve the last tweet ID for the username
-    let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
+    let last_tweet_id = read_state(&conn, None)
+        .unwrap_or_else(|e|
+            panic!("Cannot retrieve last_tweet_id: {}", e)
+        )
+        .map(|s| s.tweet_id);
 
     // get OAuth2 token
-    let token = get_oauth2_token(&config);
+    let token = get_oauth2_token(&config.twitter);
 
     // get Mastodon instance
-    let mastodon = Arc::new(Mutex::new(get_mastodon_token(&config.mastodon)));
+    let mastodon = get_mastodon_token(&config.mastodon);
 
     // get user timeline feed (Vec<tweet>)
-    let mut feed = get_user_timeline(&config, token, last_tweet_id)
+    let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
         .await
         .unwrap_or_else(|e|
         panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
@@ -357,6 +66,8 @@ pub async fn run(config: Config) {
 
     for tweet in &feed {
         debug!("Treating Tweet {} inside feed", tweet.id);
+        // initiate the toot_reply_id var
+        let mut toot_reply_id: Option<String> = None;
         // determine if the tweet is part of a thread (response to self) or a standard response
         if let Some(r) =  &tweet.in_reply_to_screen_name {
                 if &r.to_lowercase() != &config.twitter.username.to_lowercase() {
@@ -364,93 +75,77 @@ pub async fn run(config: Config) {
                     info!("Tweet is a direct response, skipping");
                     continue;
                 }
+                info!("Tweet is a thread");
+                toot_reply_id = read_state(&conn, tweet.in_reply_to_status_id)
+                    .unwrap_or(None)
+                    .map(|s| s.toot_id);
         };
 
         // build basic status by just yielding text and dereferencing contained urls
-        let mut status_text = match build_basic_status(tweet) {
+        let mut status_text = build_basic_status(tweet);
-            Ok(t) => t,
-            Err(e) => {
-                error!("Could not create status from tweet {}: {}", tweet.id ,e);
-                continue;
-            },
-        };
 
         let mut status_medias: Vec<String> = vec![];
-
+       // reupload the attachments if any
-        // reupload the attachments if any
         if let Some(m) = &tweet.extended_entities {
-            let (tx, mut rx) = mpsc::channel(4);
-
             for media in &m.media {
-                // creating a new tx for this initial loop
+                let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path).await {
-                let tx = tx.clone();
+                    Ok(m) => m,
-                // creating a new mastodon from the original mutex
+                    Err(e) => {
-                let mastodon = mastodon.clone();
+                        error!("Cannot get tweet media for {}: {}", &media.url, e);
-                // unfortunately for this to be thread safe, we need to clone a lot of structures
+                        continue;
-                let media = media.clone();
+                    },
-                let cache_path = config.scootaloo.cache_path.clone();
+                };
 
-                tokio::spawn(async move {
+                let mastodon_media_ids = match mastodon.media(Cow::from(local_tweet_media_path.to_owned())) {
-                    debug!("Spawing new async thread to treat {}", &media.id);
+                    Ok(m) => {
-                    let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
+                        remove_file(&local_tweet_media_path)
-                        Ok(m) => m,
+                            .await
-                        Err(e) => {
+                            .unwrap_or_else(|e|
-                            // we could have panicked here, no issue, but I’m not confortable using
+                                warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
-                            // that for now
-                            warn!("Cannot get tweet media for {}: {}", &media.url, e);
-                            return;
-                        }
-                    };
-
-                    // we cannot directly do all the stuff inside here because mastodon lock can
-                    // live outside this
-                    let mas_result = mastodon.lock().unwrap().media(Cow::from(String::from(&local_tweet_media_path)));
-
-                    match mas_result {
-                        Ok(m) => {
-                            remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
-                                warn!("Attachment {} has been uploaded but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
                             );
-                            // we can unwrap here because we’re in a thread
+                        m.id
-                            tx.send(ScootalooSpawnResponse {
+                    },
-                                mastodon_media_id: m.id.clone(),
+                    Err(e) => {
-                                twitter_media_url: local_tweet_media_path.clone()
+                        error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
-                            }).await.unwrap();
+                        continue;
-                        },
-                        Err(e) => {
-                            error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
-                        }
                     }
-                });
+                };
-            }
 
-            // dropping the last tx otherwise recv() will wait indefinitely
+                status_medias.push(mastodon_media_ids);
-            drop(tx);
 
-            while let Some(i) = rx.recv().await {
+                // last step, removing the reference to the media from with the toot’s text
-                // pushes the media into the media vec
+                status_text = status_text.replace(&media.url, "");
-                status_medias.push(i.mastodon_media_id);
-                // removes the URL from the original Tweet text
-                status_text = status_text.replace(&i.twitter_media_url, "");
             }
         }
         // finished reuploading attachments, now let’s do the toot baby!
 
         debug!("Building corresponding Mastodon status");
-        let status = StatusBuilder::new()
+
-            .status(&status_text)
+        let mut status_builder = StatusBuilder::new();
-            .media_ids(status_medias)
+
-            .build()
+        status_builder.status(&status_text)
-            .expect(format!("Cannot build status with text {}", &status_text).as_str());
+                .media_ids(status_medias);
+
+        if let Some(i) = toot_reply_id {
+            status_builder.in_reply_to(&i);
+        }
+
+        let status = status_builder.build()
+            .expect(&format!("Cannot build status with text {}", &status_text));
 
         // publish status
         // again unwrap is safe here as we are in the main thread
-        mastodon.lock().unwrap().new_status(status).unwrap();
+        let published_status = mastodon.new_status(status).unwrap();
         // this will panic if it cannot publish the status, which is a good thing, it allows the
         // last_tweet gathered not to be written
 
-        // write the current state (tweet ID) to avoid copying it another time
+        let ttt_towrite = TweetToToot {
-        write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e|
+            tweet_id: tweet.id,
+            toot_id: published_status.id,
+        };
+
+        // write the current state (tweet ID and toot ID) to avoid copying it another time
+        write_state(&conn, ttt_towrite).unwrap_or_else(|e|
             panic!("Can’t write the last tweet retrieved: {}", e)
         );
     }

src/main.rs

@@ -1,16 +1,11 @@
-// self
 use scootaloo::*;
-
-// clap
 use clap::{App, Arg, SubCommand};
-
-// log
 use log::{LevelFilter, error};
 use simple_logger::SimpleLogger;
-
-// std
 use std::str::FromStr;
 
+const DEFAULT_CONFIG_PATH: &'static str = "/usr/local/etc/scootaloo.toml";
+
 fn main() {
     let matches = App::new(env!("CARGO_PKG_NAME"))
                         .version(env!("CARGO_PKG_VERSION"))
@@ -19,7 +14,7 @@ fn main() {
                              .short("c")
                              .long("config")
                              .value_name("CONFIG_FILE")
-                             .help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)")
+                             .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
                              .takes_value(true)
                              .display_order(1))
                         .arg(Arg::with_name("log_level")
@@ -40,10 +35,29 @@ fn main() {
                                            .takes_value(true)
                                            .required(true)
                                            .display_order(1)))
+                             .subcommand(SubCommand::with_name("init")
+                                         .version(env!("CARGO_PKG_VERSION"))
+                                         .about("Command to init Scootaloo DB")
+                                         .arg(Arg::with_name("config")
+                                              .short("c")
+                                              .long("config")
+                                              .value_name("CONFIG_FILE")
+                                              .help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
+                                              .takes_value(true)
+                                              .display_order(1)))
                         .get_matches();
-    if let Some(matches) = matches.subcommand_matches("register") {
+
-        register(matches.value_of("host").unwrap());
+    match matches.subcommand() {
-        return;
+        ("register", Some(sub_m)) => {
+            register(sub_m.value_of("host").unwrap());
+            return;
+        },
+        ("init", Some(sub_m)) => {
+            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+            init_db(&config.scootaloo.db_path).unwrap();
+            return;
+        },
+        _ => (),
     }
 
     if matches.is_present("log_level") {
@@ -56,7 +70,7 @@ fn main() {
         };
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
+    let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
 
     run(config);
 }

src/mastodon.rs

@@ -0,0 +1,203 @@
+use crate::config::MastodonConfig;
+
+use std::{
+    borrow::Cow,
+    collections::HashMap,
+    io::stdin,
+};
+use html_escape::decode_html_entities;
+use egg_mode::{
+    tweet::Tweet,
+    entities::{UrlEntity, MentionEntity},
+};
+use elefren::{
+    prelude::*,
+    apps::App,
+    scopes::Scopes,
+};
+
+/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
+/// Fediverse
+fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
+    ums.iter().map(|s|
+        (format!("@{}", s.screen_name), format!("@{}@twitter.com", s.screen_name))
+    ).collect()
+}
+
+/// Decodes urls from UrlEntities
+fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
+    urls.iter()
+        .filter(|s| s.expanded_url.is_some())
+        .map(|s|
+            (s.url.to_owned(), s.expanded_url.as_deref().unwrap().to_owned())
+        ).collect()
+}
+
+/// Gets Mastodon Data
+pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
+    let data = Data {
+        base: Cow::from(masto.base.to_owned()),
+        client_id: Cow::from(masto.client_id.to_owned()),
+        client_secret: Cow::from(masto.client_secret.to_owned()),
+        redirect: Cow::from(masto.redirect.to_owned()),
+        token: Cow::from(masto.token.to_owned()),
+    };
+
+    Mastodon::from(data)
+}
+
+/// Builds toot text from tweet
+pub fn build_basic_status(tweet: &Tweet) -> String {
+    let mut toot = tweet.text.to_owned();
+
+    for decoded_url in decode_urls(&tweet.entities.urls) {
+        toot = toot.replace(&decoded_url.0, &decoded_url.1);
+    }
+
+    for decoded_mention in twitter_mentions(&tweet.entities.user_mentions) {
+        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
+    }
+
+    decode_html_entities(&toot).to_string()
+}
+
+/// Generic register function
+/// As this function is supposed to be run only once, it will panic for every error it encounters
+/// Most of this function is a direct copy/paste of the official `elefren` crate
+pub fn register(host: &str) {
+    let mut builder = App::builder();
+    builder.client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
+        .redirect_uris(Cow::from("urn:ietf:wg:oauth:2.0:oob".to_string()))
+        .scopes(Scopes::write_all())
+        .website(Cow::from("https://framagit.org/veretcle/scootaloo".to_string()));
+
+    let app = builder.build().expect("Cannot build the app");
+
+    let registration = Registration::new(host).register(app).expect("Cannot build registration object");
+    let url = registration.authorize_url().expect("Cannot generate registration URI!");
+
+    println!("Click this link to authorize on Mastodon: {}", url);
+    println!("Paste the returned authorization code: ");
+
+    let mut input = String::new();
+    stdin().read_line(&mut input).expect("Unable to read back registration code!");
+
+    let code = input.trim();
+    let mastodon = registration.complete(code).expect("Unable to create access token!");
+
+    let toml = toml::to_string(&*mastodon).unwrap();
+
+    println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use egg_mode::tweet::TweetEntities;
+    use chrono::prelude::*;
+
+    #[test]
+    fn test_twitter_mentions() {
+        let mention_entity = MentionEntity {
+            id: 12345,
+            range: (1, 3),
+            name: "Ta Mere l0l".to_string(),
+            screen_name: "tamerelol".to_string(),
+        };
+
+        let twitter_ums = vec![mention_entity];
+
+        let mut expected_mentions = HashMap::new();
+        expected_mentions.insert("@tamerelol".to_string(), "@tamerelol@twitter.com".to_string());
+
+        let decoded_mentions = twitter_mentions(&twitter_ums);
+
+        assert_eq!(expected_mentions, decoded_mentions);
+    }
+
+    #[test]
+    fn test_decode_urls() {
+        let url_entity1 = UrlEntity {
+            display_url: "tamerelol".to_string(),
+            expanded_url: Some("https://www.nintendojo.fr/dojobar".to_string()),
+            range: (1, 3),
+            url: "https://t.me/tamerelol".to_string(),
+        };
+
+        let url_entity2 = UrlEntity {
+            display_url: "tamerelol".to_string(),
+            expanded_url: None,
+            range: (1, 3),
+            url: "https://t.me/tamerelol".to_string(),
+        };
+
+        let twitter_urls = vec![url_entity1, url_entity2];
+
+        let mut expected_urls = HashMap::new();
+        expected_urls.insert("https://t.me/tamerelol".to_string(), "https://www.nintendojo.fr/dojobar".to_string());
+
+        let decoded_urls = decode_urls(&twitter_urls);
+
+        assert_eq!(expected_urls, decoded_urls);
+    }
+
+    #[test]
+    fn test_build_basic_status() {
+        let t = Tweet {
+            coordinates: None,
+            created_at: Utc::now(),
+            current_user_retweet: None,
+            display_text_range: None,
+            entities: TweetEntities {
+                hashtags: vec![],
+                symbols: vec![],
+                urls: vec![
+                        UrlEntity {
+                            display_url: "youtube.com/watch?v=w5TrSa…".to_string(),
+                            expanded_url: Some("https://www.youtube.com/watch?v=w5TrSaoYmZ8".to_string()),
+                            range: (93, 116),
+                            url: "https://t.co/zXw0FfX2Nt".to_string(),
+                        }
+                ],
+                user_mentions: vec![
+                        MentionEntity {
+                            id: 491500016,
+                            range: (80, 95),
+                            name: "Nintendo France".to_string(),
+                            screen_name: "NintendoFrance".to_string(),
+                        }
+                    ],
+                media: None,
+            },
+            extended_entities: None,
+            favorite_count: 0,
+            favorited: None,
+            filter_level: None,
+            id: 1491541246984306693,
+            in_reply_to_user_id: None,
+            in_reply_to_screen_name: None,
+            in_reply_to_status_id: None,
+            lang: None,
+            place: None,
+            possibly_sensitive: None,
+            quoted_status: None,
+            quoted_status_id: None,
+            retweet_count: 0,
+            retweeted: None,
+            retweeted_status: None,
+            source: None,
+            text: "Mother 1 &amp; 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance https://t.co/zXw0FfX2Nt".to_string(),
+            truncated: false,
+            user: None,
+            withheld_copyright: false,
+            withheld_in_countries: None,
+            withheld_scope: None,
+        };
+
+        let t_out = build_basic_status(&t);
+
+        assert_eq!(&t_out, "Mother 1 & 2 sur le NES/SNES online !\nDispo maintenant. cc @NintendoFrance@twitter.com https://www.youtube.com/watch?v=w5TrSaoYmZ8");
+    }
+}
+

src/state.rs

@@ -0,0 +1,223 @@
+use std::error::Error;
+use log::debug;
+use rusqlite::{Connection, params, OptionalExtension};
+
+/// Struct for each query line
+#[derive(Debug)]
+pub struct TweetToToot {
+    pub tweet_id: u64,
+    pub toot_id: String,
+}
+
+/// if None is passed, read the last tweet from DB
+/// if a tweet_id is passed, read this particular tweet from DB
+pub fn read_state(conn: &Connection, s: Option<u64>) -> Result<Option<TweetToToot>, Box<dyn Error>> {
+    debug!("Reading tweet_id {:?}", s);
+    let query: String;
+    match s {
+        Some(i) => query = format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {}", i),
+        None => query = "SELECT * FROM tweet_to_toot ORDER BY tweet_id DESC LIMIT 1".to_string(),
+    };
+
+    let mut stmt = conn.prepare(&query)?;
+
+    let t = stmt.query_row([], |row| {
+        Ok(TweetToToot {
+            tweet_id: row.get(0)?,
+            toot_id: row.get(1)?,
+        })
+    }).optional()?;
+
+    Ok(t)
+}
+
+/// Writes last treated tweet id and toot id to the db
+pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
+    debug!("Write struct {:?}", t);
+    conn.execute(
+        "INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)",
+        params![t.tweet_id, t.toot_id],
+        )?;
+
+    Ok(())
+}
+
+/// Initiates the DB from path
+pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
+    debug!("Initializing DB for Scootaloo");
+    let conn = Connection::open(d)?;
+
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS tweet_to_toot (
+            tweet_id        INTEGER PRIMARY KEY,
+            toot_id         TEXT UNIQUE
+            )",
+        [],
+    )?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::{
+        fs::remove_file,
+        path::Path,
+    };
+
+    #[test]
+    fn test_init_db() {
+        let d = "/tmp/test_init_db.sqlite";
+
+        init_db(d).unwrap();
+
+        // check that file exist
+        assert!(Path::new(d).exists());
+
+        // open said file
+        let conn = Connection::open(d).unwrap();
+        conn.execute(
+            "SELECT * from tweet_to_toot;",
+            [],
+        ).unwrap();
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_init_init_db() {
+        // init_db fn should be idempotent so let’s test that
+        let d = "/tmp/test_init_init_db.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot
+            VALUES
+            (100, 'A');",
+            [],
+        ).unwrap();
+
+        init_db(d).unwrap();
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_write_state() {
+        let d = "/tmp/test_write_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        let t_in = TweetToToot {
+            tweet_id: 123456789,
+            toot_id: "987654321".to_string(),
+        };
+
+        write_state(&conn, t_in).unwrap();
+
+        let mut stmt = conn.prepare("SELECT * FROM tweet_to_toot;").unwrap();
+
+        let t_out = stmt.query_row([], |row| {
+            Ok(TweetToToot {
+                tweet_id: row.get(0).unwrap(),
+                toot_id: row.get(1).unwrap(),
+            })
+        }).unwrap();
+
+        assert_eq!(t_out.tweet_id, 123456789);
+        assert_eq!(t_out.toot_id, "987654321".to_string());
+
+        remove_file(d).unwrap();
+    }
+
+    #[test]
+    fn test_none_to_tweet_id_read_state() {
+        let d = "/tmp/test_none_to_tweet_id_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            VALUES
+            (101, 'A'),
+            (102, 'B');",
+            [],
+            ).unwrap();
+
+        let t_out = read_state(&conn, None).unwrap().unwrap();
+
+        remove_file(d).unwrap();
+
+        assert_eq!(t_out.tweet_id, 102);
+        assert_eq!(t_out.toot_id, "B");
+    }
+
+    #[test]
+    fn test_none_to_none_read_state() {
+        let d = "/tmp/test_none_to_none_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        let t_out = read_state(&conn, None).unwrap();
+
+        remove_file(d).unwrap();
+
+        assert!(t_out.is_none());
+    }
+
+    #[test]
+    fn test_tweet_id_to_none_read_state() {
+        let d = "/tmp/test_tweet_id_to_none_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            VALUES
+            (100, 'A');",
+            [],
+            ).unwrap();
+
+        let t_out = read_state(&conn, Some(101)).unwrap();
+
+        remove_file(d).unwrap();
+
+        assert!(t_out.is_none());
+    }
+
+    #[test]
+    fn test_tweet_id_to_tweet_id_read_state() {
+        let d = "/tmp/test_tweet_id_to_tweet_id_read_state.sqlite";
+
+        init_db(d).unwrap();
+
+        let conn = Connection::open(d).unwrap();
+
+        conn.execute(
+            "INSERT INTO tweet_to_toot (tweet_id, toot_id)
+            VALUES
+            (100, 'A');",
+            [],
+            ).unwrap();
+
+        let t_out = read_state(&conn, Some(100)).unwrap().unwrap();
+
+        remove_file(d).unwrap();
+
+        assert_eq!(t_out.tweet_id, 100);
+        assert_eq!(t_out.toot_id, "A");
+    }
+}
+

src/twitter.rs

@@ -0,0 +1,62 @@
+use crate::ScootalooError;
+use crate::config::TwitterConfig;
+use crate::util::cache_media;
+
+use std::error::Error;
+use egg_mode::{
+    Token,
+    KeyPair,
+    entities::{MediaEntity, MediaType},
+    user::UserID,
+    tweet::{
+        Tweet,
+        user_timeline,
+    },
+};
+
+/// Gets Twitter oauth2 token
+pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
+    let con_token = KeyPair::new(config.consumer_key.to_owned(),config.consumer_secret.to_owned());
+    let access_token = KeyPair::new(config.access_key.to_owned(), config.access_secret.to_owned());
+
+    Token::Access {
+        consumer: con_token,
+        access: access_token,
+    }
+}
+
+/// Gets Twitter user timeline
+pub async fn get_user_timeline(config: &TwitterConfig, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
+    // fix the page size to 200 as it is the maximum Twitter authorizes
+    let (_, feed) = user_timeline(UserID::from(config.username.to_owned()), true, false, &token)
+                .with_page_size(200)
+                .older(lid)
+                .await?;
+
+    Ok(feed.to_vec())
+}
+
+/// Retrieves a single media from a tweet and store it in a temporary file
+pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
+    match m.media_type {
+        MediaType::Photo => {
+            return cache_media(&m.media_url_https, t).await;
+        },
+        _ => {
+            match &m.video_info {
+                Some(v) => {
+                    for variant in &v.variants {
+                        if variant.content_type == "video/mp4" {
+                            return cache_media(&variant.url, t).await;
+                        }
+                    }
+                    return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
+                },
+                None => {
+                    return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
+                },
+            }
+        },
+    };
+}
+

src/util.rs

@@ -0,0 +1,53 @@
+use std::error::Error;
+use crate::ScootalooError;
+use reqwest::Url;
+use tokio::{
+    io::copy,
+    fs::{File, create_dir_all},
+};
+
+/// Gets and caches Twitter Media inside the determined temp dir
+pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
+    // create dir
+    create_dir_all(t).await?;
+
+    // get file
+    let mut response = reqwest::get(u).await?;
+
+    // create local file
+    let url = Url::parse(u)?;
+    let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
+                    .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
+
+    let dest_filepath = format!("{}/{}", t, dest_filename);
+
+    let mut dest_file = File::create(&dest_filepath).await?;
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut dest_file).await?;
+    }
+
+    Ok(dest_filepath)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::{
+        path::Path,
+        fs::remove_dir_all,
+    };
+
+    const TMP_DIR: &'static str = "/tmp/scootaloo_test";
+
+    #[tokio::test]
+    async fn test_cache_media() {
+        let dest = cache_media("https://forum.nintendojo.fr/styles/prosilver/theme/images/ndfr_casual.png", TMP_DIR).await.unwrap();
+
+        assert!(Path::new(&dest).exists());
+
+        remove_dir_all(TMP_DIR).unwrap();
+    }
+}
+

tests/bad_test.toml

@@ -0,0 +1 @@
+blah

tests/config.rs

@@ -0,0 +1,33 @@
+use scootaloo::parse_toml;
+
+#[test]
+fn test_parse_good_toml() {
+    let parse_good_toml = parse_toml("tests/good_test.toml");
+
+    assert_eq!(parse_good_toml.scootaloo.db_path, "/var/random/scootaloo.sqlite");
+    assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
+
+    assert_eq!(parse_good_toml.twitter.username, "tamerelol");
+    assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
+    assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");
+    assert_eq!(parse_good_toml.twitter.access_key, "rand access key");
+    assert_eq!(parse_good_toml.twitter.access_secret, "super secret");
+
+    assert_eq!(parse_good_toml.mastodon.base, "https://m.nintendojo.fr");
+    assert_eq!(parse_good_toml.mastodon.client_id, "rand client id");
+    assert_eq!(parse_good_toml.mastodon.client_secret, "secret");
+    assert_eq!(parse_good_toml.mastodon.redirect, "urn:ietf:wg:oauth:2.0:oob");
+    assert_eq!(parse_good_toml.mastodon.token, "super secret");
+}
+
+#[test]
+#[should_panic(expected = "Cannot open config file tests/no_file.toml: No such file or directory (os error 2)")]
+fn test_parse_no_toml() {
+    let _parse_no_toml = parse_toml("tests/no_file.toml");
+}
+
+#[test]
+#[should_panic(expected = "Cannot parse TOML file tests/bad_test.toml: expected an equals, found a newline at line 1 column 5")]
+fn test_parse_bad_toml() {
+    let _parse_bad_toml = parse_toml("tests/bad_test.toml");
+}

tests/good_test.toml

@@ -0,0 +1,19 @@
+[scootaloo]
+
+db_path="/var/random/scootaloo.sqlite"
+cache_path="/tmp/scootaloo"
+
+[twitter]
+username="tamerelol"
+
+consumer_key="rand consumer key"
+consumer_secret="secret"
+access_key="rand access key"
+access_secret="super secret"
+
+[mastodon]
+base = "https://m.nintendojo.fr"
+client_id = "rand client id"
+client_secret = "secret"
+redirect = "urn:ietf:wg:oauth:2.0:oob"
+token = "super secret"