refactor: make everything a little more modular

Cargo.lock

@@ -17,6 +17,17 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.6",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.15"
@@ -550,6 +561,18 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
 [[package]]
 name = "flate2"
 version = "1.0.20"
@@ -755,9 +778,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.2"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
+checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
@@ -819,6 +842,24 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
+dependencies = [
+ "hashbrown 0.11.2",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.18"
@@ -1027,7 +1068,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
 dependencies = [
  "autocfg 1.0.1",
- "hashbrown",
+ "hashbrown 0.9.1",
 ]
 
 [[package]]
@@ -1123,6 +1164,16 @@ version = "0.2.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
+dependencies = [
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.3.4"
@@ -1340,9 +1391,9 @@ checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
 
 [[package]]
 name = "once_cell"
-version = "1.7.2"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
+checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
 
 [[package]]
 name = "opaque-debug"
@@ -1698,7 +1749,7 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
 dependencies = [
- "getrandom 0.2.2",
+ "getrandom 0.2.6",
 ]
 
 [[package]]
@@ -1899,6 +1950,21 @@ dependencies = [
  "winreg 0.7.0",
 ]
 
+[[package]]
+name = "rusqlite"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
+dependencies = [
+ "bitflags",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "memchr",
+ "smallvec 1.6.1",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.18"
@@ -1947,7 +2013,7 @@ dependencies = [
 
 [[package]]
 name = "scootaloo"
-version = "0.4.2"
+version = "0.5.0"
 dependencies = [
  "clap",
  "egg-mode",
@@ -1956,6 +2022,7 @@ dependencies = [
  "htmlescape",
  "log",
  "reqwest 0.11.3",
+ "rusqlite",
  "serde",
  "simple_logger",
  "tokio 1.5.0",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scootaloo"
-version = "0.4.2"
+version = "0.5.0"
 authors = ["VC <veretcle+framagit@mateu.be>"]
 edition = "2021"
 
@@ -12,6 +12,7 @@ toml = "^0.5"
 clap = "^2.34"
 futures = "^0.3"
 egg-mode = "^0.16"
+rusqlite = "^0.27"
 tokio = { version = "1", features = ["full"]}
 elefren = "^0.22"
 htmlescape = "^0.3"

README.md

@@ -7,7 +7,7 @@ It:
 
 If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
 
-RT are excluded, replies are included.but only the source threads are copied, not the actual replies to other Twitter users.
+RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
 
 # Usage
 
@@ -16,7 +16,7 @@ First up, create a configuration file (default path is `/usr/local/etc/scootaloo
 ```toml
 [scootaloo]
 
-last_tweet_path="/usr/local/etc/last_tweet" ## file containing the last tweet id received, must be writable
+db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
 cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
 
 [twitter]
@@ -29,6 +29,11 @@ access_key="MYACCESSKEY"
 access_secret="MYACCESSSECRET"
 ```
 
+Then run the command with the `init` subcommand to initiate the DB:
+```
+scootaloo init
+```
+
 Then run the command with the `register` subcommand:
 ```sh
 scootaloo register --host https://m.nintendojo.fr

src/config.rs

@@ -0,0 +1,51 @@
+// std
+use std::fs::read_to_string;
+
+// toml
+use serde::Deserialize;
+
+/// General configuration Struct
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub twitter: TwitterConfig,
+    pub mastodon: MastodonConfig,
+    pub scootaloo: ScootalooConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TwitterConfig {
+    pub username: String,
+    pub consumer_key: String,
+    pub consumer_secret: String,
+    pub access_key: String,
+    pub access_secret: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct MastodonConfig {
+    pub base: String,
+    pub client_id: String,
+    pub client_secret: String,
+    pub redirect: String,
+    pub token: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ScootalooConfig {
+    pub db_path: String,
+    pub cache_path: String,
+}
+
+/// Parses the TOML file into a Config Struct
+pub fn parse_toml(toml_file: &str) -> Config {
+    let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
+        panic!("Cannot open config file {}: {}", toml_file, e)
+    );
+
+    let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
+        panic!("Cannot parse TOML file {}: {}", toml_file, e)
+    );
+
+    config
+}
+

src/error.rs

@@ -0,0 +1,27 @@
+use std::fmt;
+
+#[derive(Debug)]
+pub struct ScootalooError {
+    details: String,
+}
+
+impl ScootalooError {
+    pub fn new(msg: &str) -> ScootalooError {
+        ScootalooError {
+            details: String::from(msg),
+        }
+    }
+}
+
+impl fmt::Display for ScootalooError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.details)
+    }
+}
+
+impl std::error::Error for ScootalooError {
+    fn description(&self) -> &str {
+        &self.details
+    }
+}
+

src/lib.rs

@@ -1,334 +1,53 @@
+// auto-imports
+mod error;
+use error::ScootalooError;
+
+mod config;
+use config::Config;
+pub use config::parse_toml;
+
+mod mastodon;
+use mastodon::{get_mastodon_token, build_basic_status};
+pub use mastodon::register;
+
+mod twitter;
+use twitter::*;
+
+mod util;
+
+mod state;
+use state::{read_state, write_state};
+pub use state::init_db;
+
 // std
-use std::{
+use std::borrow::Cow;
-    borrow::Cow,
-    collections::HashMap,
-    io::stdin,
-    fmt,
-    fs::{read_to_string, write},
-    error::Error,
-};
 
-// toml
+// tokio
-use serde::Deserialize;
+use tokio::fs::remove_file;
-
-// egg-mode
-use egg_mode::{
-    Token,
-    KeyPair,
-    entities::{UrlEntity, MediaEntity, MentionEntity, MediaType},
-    user::UserID,
-    tweet::{
-        Tweet,
-        user_timeline,
-    },
-};
 
 // elefren
 use elefren::{
     prelude::*,
-    apps::App,
     status_builder::StatusBuilder,
-    scopes::Scopes,
 };
 
-// reqwest
-use reqwest::Url;
-
-// tokio
-use tokio::{
-    io::copy,
-    fs::{File, create_dir_all, remove_file},
-};
-
-// htmlescape
-use htmlescape::decode_html;
-
 // log
 use log::{info, warn, error, debug};
 
-/**********
- * Generic usage functions
-***********/
-/*
- * Those functions are related to the Twitter side of things
- */
-/// Reads last tweet id from a file
-fn read_state(s: &str) -> Option<u64> {
-    let state = read_to_string(s);
-
-    if let Ok(s) = state {
-        debug!("Last Tweet ID (from file): {}", &s);
-        return s.parse::<u64>().ok();
-    }
-
-    None
-}
-
-/// Writes last treated tweet id to a file
-fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
-    write(f, format!("{}", s))
-}
-
-/// Gets Twitter oauth2 token
-fn get_oauth2_token(config: &Config) -> Token {
-    let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
-    let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
-
-    Token::Access {
-        consumer: con_token,
-        access: access_token,
-    }
-}
-
-/// Gets Twitter user timeline
-async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
-    // fix the page size to 200 as it is the maximum Twitter authorizes
-    let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
-                .with_page_size(200)
-                .older(lid)
-                .await?;
-
-    Ok(feed.to_vec())
-}
-
-/// Decodes urls from UrlEntities
-fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
-    let mut decoded_urls = HashMap::new();
-
-    for url in urls {
-        if url.expanded_url.is_some() {
-            // unwrap is safe here as we just verified that there is something inside expanded_url
-            decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
-        }
-    }
-
-    decoded_urls
-}
-
-/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
-/// Fediverse
-fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
-    let mut decoded_mentions = HashMap::new();
-
-    for um in ums {
-        decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
-    }
-
-    decoded_mentions
-}
-
-/// Retrieves a single media from a tweet and store it in a temporary file
-async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
-    match m.media_type {
-        MediaType::Photo => {
-            return cache_media(&m.media_url_https, t).await;
-        },
-        _ => {
-            match &m.video_info {
-                Some(v) => {
-                    for variant in &v.variants {
-                        if variant.content_type == "video/mp4" {
-                            return cache_media(&variant.url, t).await;
-                        }
-                    }
-                    return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
-                },
-                None => {
-                    return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
-                },
-            }
-        },
-    };
-}
-
-/*
- * Those functions are related to the Mastodon side of things
- */
-/// Gets Mastodon Data
-fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
-    let data = Data {
-        base: Cow::from(String::from(&masto.base)),
-        client_id: Cow::from(String::from(&masto.client_id)),
-        client_secret: Cow::from(String::from(&masto.client_secret)),
-        redirect: Cow::from(String::from(&masto.redirect)),
-        token: Cow::from(String::from(&masto.token)),
-    };
-
-    Mastodon::from(data)
-}
-
-/// Builds toot text from tweet
-fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
-    let mut toot = String::from(&tweet.text);
-
-    let decoded_urls = decode_urls(&tweet.entities.urls);
-
-    for decoded_url in decoded_urls {
-        toot = toot.replace(&decoded_url.0, &decoded_url.1);
-    }
-
-    let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
-
-    for decoded_mention in decoded_mentions {
-        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
-    }
-
-    if let Ok(t) = decode_html(&toot) {
-        toot = t;
-    }
-
-    Ok(toot)
-}
-
-/*
- * Generic private functions
- */
-/// Gets and caches Twitter Media inside the determined temp dir
-async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
-    // create dir
-    create_dir_all(t).await?;
-
-    // get file
-    let mut response = reqwest::get(u).await?;
-
-    // create local file
-    let url = Url::parse(u)?;
-    let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
-                    .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
-
-    let dest_filepath = format!("{}/{}", t, dest_filename);
-
-    let mut dest_file = File::create(&dest_filepath).await?;
-
-    while let Some(chunk) = response.chunk().await? {
-        copy(&mut &*chunk, &mut dest_file).await?;
-    }
-
-    Ok(dest_filepath)
-}
-
-/**********
- * local error handler
-**********/
-#[derive(Debug)]
-struct ScootalooError {
-    details: String,
-}
-
-impl ScootalooError {
-    fn new(msg: &str) -> ScootalooError {
-        ScootalooError {
-            details: String::from(msg),
-        }
-    }
-}
-
-impl fmt::Display for ScootalooError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.details)
-    }
-}
-
-impl std::error::Error for ScootalooError {
-    fn description(&self) -> &str {
-        &self.details
-    }
-}
-
-/**********
- * Config structure
-***********/
-/// General configuration Struct
-#[derive(Debug, Deserialize)]
-pub struct Config {
-    twitter: TwitterConfig,
-    mastodon: MastodonConfig,
-    scootaloo: ScootalooConfig,
-}
-
-#[derive(Debug, Deserialize)]
-struct TwitterConfig {
-    username: String,
-    consumer_key: String,
-    consumer_secret: String,
-    access_key: String,
-    access_secret: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct MastodonConfig {
-    base: String,
-    client_id: String,
-    client_secret: String,
-    redirect: String,
-    token: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct ScootalooConfig {
-    last_tweet_path: String,
-    cache_path: String,
-}
-
-/*********
- * Main functions
-*********/
-/// Parses the TOML file into a Config Struct
-pub fn parse_toml(toml_file: &str) -> Config {
-    let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
-        panic!("Cannot open config file {}: {}", toml_file, e)
-    );
-
-    let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
-        panic!("Cannot parse TOML file {}: {}", toml_file, e)
-    );
-
-    config
-}
-
-/// Generic register function
-/// As this function is supposed to be run only once, it will panic for every error it encounters
-/// Most of this function is a direct copy/paste of the official `elefren` crate
-pub fn register(host: &str) {
-    let mut builder = App::builder();
-    builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
-        .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
-        .scopes(Scopes::write_all())
-        .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
-
-    let app = builder.build().expect("Cannot build the app");
-
-    let registration = Registration::new(host).register(app).expect("Cannot build registration object");
-    let url = registration.authorize_url().expect("Cannot generate registration URI!");
-
-    println!("Click this link to authorize on Mastodon: {}", url);
-    println!("Paste the returned authorization code: ");
-
-    let mut input = String::new();
-    stdin().read_line(&mut input).expect("Unable to read back registration code!");
-
-    let code = input.trim();
-    let mastodon = registration.complete(code).expect("Unable to create access token!");
-
-    let toml = toml::to_string(&*mastodon).unwrap();
-
-    println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
-}
-
 /// This is where the magic happens
 #[tokio::main]
 pub async fn run(config: Config) {
     // retrieve the last tweet ID for the username
-    let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
+    let last_tweet_id = read_state(&config.scootaloo.db_path);
 
     // get OAuth2 token
-    let token = get_oauth2_token(&config);
+    let token = get_oauth2_token(&config.twitter);
 
     // get Mastodon instance
     let mastodon = get_mastodon_token(&config.mastodon);
 
     // get user timeline feed (Vec<tweet>)
-    let mut feed = get_user_timeline(&config, token, last_tweet_id)
+    let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
         .await
         .unwrap_or_else(|e|
         panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
@@ -411,7 +130,7 @@ pub async fn run(config: Config) {
         // last_tweet gathered not to be written
 
         // write the current state (tweet ID) to avoid copying it another time
-        write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e|
+        write_state(&config.scootaloo.db_path, tweet.id).unwrap_or_else(|e|
             panic!("Can’t write the last tweet retrieved: {}", e)
         );
     }

src/main.rs

@@ -11,6 +11,8 @@ use simple_logger::SimpleLogger;
 // std
 use std::str::FromStr;
 
+const DEFAULT_CONFIG_PATH: &'static str = "/usr/local/etc/scootaloo.toml";
+
 fn main() {
     let matches = App::new(env!("CARGO_PKG_NAME"))
                         .version(env!("CARGO_PKG_VERSION"))
@@ -40,10 +42,29 @@ fn main() {
                                            .takes_value(true)
                                            .required(true)
                                            .display_order(1)))
+                             .subcommand(SubCommand::with_name("init")
+                                         .version(env!("CARGO_PKG_VERSION"))
+                                         .about("Command to init Scootaloo DB")
+                                         .arg(Arg::with_name("config")
+                                              .short("c")
+                                              .long("config")
+                                              .value_name("CONFIG_FILE")
+                                              .help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml")
+                                              .takes_value(true)
+                                              .display_order(1)))
                         .get_matches();
-    if let Some(matches) = matches.subcommand_matches("register") {
+
-        register(matches.value_of("host").unwrap());
+    match matches.subcommand() {
+        ("register", Some(sub_m)) => {
+            register(sub_m.value_of("host").unwrap());
             return;
+        },
+        ("init", Some(sub_m)) => {
+            let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
+            init_db(&config).unwrap();
+            return;
+        },
+        _ => (),
     }
 
     if matches.is_present("log_level") {
@@ -56,7 +77,7 @@ fn main() {
         };
     }
 
-    let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
+    let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
 
     run(config);
 }

src/mastodon.rs

@@ -0,0 +1,106 @@
+// auto imports
+use crate::config::MastodonConfig;
+use crate::twitter::decode_urls;
+
+// std
+use std::{
+    borrow::Cow,
+    error::Error,
+    collections::HashMap,
+    io::stdin,
+};
+
+// htmlescape
+use htmlescape::decode_html;
+
+// egg-mode
+use egg_mode::{
+    tweet::Tweet,
+    entities::MentionEntity,
+};
+
+// elefren
+use elefren::{
+    prelude::*,
+    apps::App,
+    scopes::Scopes,
+};
+
+
+/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
+/// Fediverse
+fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
+    let mut decoded_mentions = HashMap::new();
+
+    for um in ums {
+        decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
+    }
+
+    decoded_mentions
+}
+
+/// Gets Mastodon Data
+pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
+    let data = Data {
+        base: Cow::from(String::from(&masto.base)),
+        client_id: Cow::from(String::from(&masto.client_id)),
+        client_secret: Cow::from(String::from(&masto.client_secret)),
+        redirect: Cow::from(String::from(&masto.redirect)),
+        token: Cow::from(String::from(&masto.token)),
+    };
+
+    Mastodon::from(data)
+}
+
+/// Builds toot text from tweet
+pub fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
+    let mut toot = String::from(&tweet.text);
+
+    let decoded_urls = decode_urls(&tweet.entities.urls);
+
+    for decoded_url in decoded_urls {
+        toot = toot.replace(&decoded_url.0, &decoded_url.1);
+    }
+
+    let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
+
+    for decoded_mention in decoded_mentions {
+        toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
+    }
+
+    if let Ok(t) = decode_html(&toot) {
+        toot = t;
+    }
+
+    Ok(toot)
+}
+
+/// Generic register function
+/// As this function is supposed to be run only once, it will panic for every error it encounters
+/// Most of this function is a direct copy/paste of the official `elefren` crate
+pub fn register(host: &str) {
+    let mut builder = App::builder();
+    builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
+        .redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
+        .scopes(Scopes::write_all())
+        .website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
+
+    let app = builder.build().expect("Cannot build the app");
+
+    let registration = Registration::new(host).register(app).expect("Cannot build registration object");
+    let url = registration.authorize_url().expect("Cannot generate registration URI!");
+
+    println!("Click this link to authorize on Mastodon: {}", url);
+    println!("Paste the returned authorization code: ");
+
+    let mut input = String::new();
+    stdin().read_line(&mut input).expect("Unable to read back registration code!");
+
+    let code = input.trim();
+    let mastodon = registration.complete(code).expect("Unable to create access token!");
+
+    let toml = toml::to_string(&*mastodon).unwrap();
+
+    println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
+}
+

src/state.rs

@@ -0,0 +1,38 @@
+// auto-imports
+use crate::config::Config;
+
+// std
+use std::{
+    fs::{read_to_string, write},
+    error::Error,
+};
+
+// log
+use log::debug;
+
+/// Reads last tweet id from a file
+pub fn read_state(s: &str) -> Option<u64> {
+    let state = read_to_string(s);
+
+    if let Ok(s) = state {
+        debug!("Last Tweet ID (from file): {}", &s);
+        return s.parse::<u64>().ok();
+    }
+
+    None
+}
+
+/// Writes last treated tweet id to a file
+pub fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
+    write(f, format!("{}", s))
+}
+
+/*********
+ * Main functions
+*********/
+/// Initiates the DB from path
+pub fn init_db(config: &Config) -> Result<(), Box<dyn Error>> {
+    println!("config.scootaloo.db_path: {}", config.scootaloo.db_path);
+    Ok(())
+}
+

src/twitter.rs

@@ -0,0 +1,83 @@
+// auto-imports
+use crate::ScootalooError;
+use crate::config::TwitterConfig;
+use crate::util::cache_media;
+
+// std
+use std::{
+    error::Error,
+    collections::HashMap,
+};
+
+// egg-mode
+use egg_mode::{
+    Token,
+    KeyPair,
+    entities::{UrlEntity, MediaEntity, MediaType},
+    user::UserID,
+    tweet::{
+        Tweet,
+        user_timeline,
+    },
+};
+
+/// Gets Twitter oauth2 token
+pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
+    let con_token = KeyPair::new(String::from(&config.consumer_key), String::from(&config.consumer_secret));
+    let access_token = KeyPair::new(String::from(&config.access_key), String::from(&config.access_secret));
+
+    Token::Access {
+        consumer: con_token,
+        access: access_token,
+    }
+}
+
+/// Gets Twitter user timeline
+pub async fn get_user_timeline(config: &TwitterConfig, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
+    // fix the page size to 200 as it is the maximum Twitter authorizes
+    let (_, feed) = user_timeline(UserID::from(String::from(&config.username)), true, false, &token)
+                .with_page_size(200)
+                .older(lid)
+                .await?;
+
+    Ok(feed.to_vec())
+}
+
+/// Decodes urls from UrlEntities
+pub fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
+    let mut decoded_urls = HashMap::new();
+
+    for url in urls {
+        if url.expanded_url.is_some() {
+            // unwrap is safe here as we just verified that there is something inside expanded_url
+            decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
+        }
+    }
+
+    decoded_urls
+}
+
+/// Retrieves a single media from a tweet and store it in a temporary file
+pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
+    match m.media_type {
+        MediaType::Photo => {
+            return cache_media(&m.media_url_https, t).await;
+        },
+        _ => {
+            match &m.video_info {
+                Some(v) => {
+                    for variant in &v.variants {
+                        if variant.content_type == "video/mp4" {
+                            return cache_media(&variant.url, t).await;
+                        }
+                    }
+                    return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
+                },
+                None => {
+                    return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
+                },
+            }
+        },
+    };
+}
+

src/util.rs

@@ -0,0 +1,37 @@
+// std
+use std::error::Error;
+use crate::ScootalooError;
+
+// reqwest
+use reqwest::Url;
+
+// tokio
+use tokio::{
+    io::copy,
+    fs::{File, create_dir_all},
+};
+
+/// Gets and caches Twitter Media inside the determined temp dir
+pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
+    // create dir
+    create_dir_all(t).await?;
+
+    // get file
+    let mut response = reqwest::get(u).await?;
+
+    // create local file
+    let url = Url::parse(u)?;
+    let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
+                    .last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
+
+    let dest_filepath = format!("{}/{}", t, dest_filename);
+
+    let mut dest_file = File::create(&dest_filepath).await?;
+
+    while let Some(chunk) = response.chunk().await? {
+        copy(&mut &*chunk, &mut dest_file).await?;
+    }
+
+    Ok(dest_filepath)
+}
+