mirror of
https://framagit.org/veretcle/scootaloo.git
synced 2025-07-20 17:11:19 +02:00
refactor: make everything a little more modular
This commit is contained in:
VC
81
Cargo.lock
generated
81
Cargo.lock
generated
@@ -17,6 +17,17 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
|
||||
dependencies = [
|
||||
"getrandom 0.2.6",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.15"
|
||||
@@ -550,6 +561,18 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.20"
|
||||
@@ -755,9 +778,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.2"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
|
||||
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"libc",
|
||||
@@ -819,6 +842,24 @@ version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
|
||||
dependencies = [
|
||||
"hashbrown 0.11.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.18"
|
||||
@@ -1027,7 +1068,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"hashbrown",
|
||||
"hashbrown 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1123,6 +1164,16 @@ version = "0.2.124"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
|
||||
dependencies = [
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.3.4"
|
||||
@@ -1340,9 +1391,9 @@ checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.7.2"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
|
||||
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
@@ -1698,7 +1749,7 @@ version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
|
||||
dependencies = [
|
||||
"getrandom 0.2.2",
|
||||
"getrandom 0.2.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1899,6 +1950,21 @@ dependencies = [
|
||||
"winreg 0.7.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"memchr",
|
||||
"smallvec 1.6.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.18"
|
||||
@@ -1947,7 +2013,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "scootaloo"
|
||||
version = "0.4.2"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"egg-mode",
|
||||
@@ -1956,6 +2022,7 @@ dependencies = [
|
||||
"htmlescape",
|
||||
"log",
|
||||
"reqwest 0.11.3",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"simple_logger",
|
||||
"tokio 1.5.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "scootaloo"
|
||||
version = "0.4.2"
|
||||
version = "0.5.0"
|
||||
authors = ["VC <veretcle+framagit@mateu.be>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -12,6 +12,7 @@ toml = "^0.5"
|
||||
clap = "^2.34"
|
||||
futures = "^0.3"
|
||||
egg-mode = "^0.16"
|
||||
rusqlite = "^0.27"
|
||||
tokio = { version = "1", features = ["full"]}
|
||||
elefren = "^0.22"
|
||||
htmlescape = "^0.3"
|
||||
|
||||
@@ -7,7 +7,7 @@ It:
|
||||
|
||||
If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
|
||||
|
||||
RT are excluded, replies are included.but only the source threads are copied, not the actual replies to other Twitter users.
|
||||
RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
|
||||
|
||||
# Usage
|
||||
|
||||
@@ -16,7 +16,7 @@ First up, create a configuration file (default path is `/usr/local/etc/scootaloo
|
||||
```toml
|
||||
[scootaloo]
|
||||
|
||||
last_tweet_path="/usr/local/etc/last_tweet" ## file containing the last tweet id received, must be writable
|
||||
db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
|
||||
cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
|
||||
|
||||
[twitter]
|
||||
@@ -29,6 +29,11 @@ access_key="MYACCESSKEY"
|
||||
access_secret="MYACCESSSECRET"
|
||||
```
|
||||
|
||||
Then run the command with the `init` subcommand to initiate the DB:
|
||||
```
|
||||
scootaloo init
|
||||
```
|
||||
|
||||
Then run the command with the `register` subcommand:
|
||||
```sh
|
||||
scootaloo register --host https://m.nintendojo.fr
|
||||
|
||||
51
src/config.rs
Normal file
51
src/config.rs
Normal file
@@ -0,0 +1,51 @@
|
||||
// std
|
||||
use std::fs::read_to_string;
|
||||
|
||||
// toml
|
||||
use serde::Deserialize;
|
||||
|
||||
/// General configuration Struct
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
pub twitter: TwitterConfig,
|
||||
pub mastodon: MastodonConfig,
|
||||
pub scootaloo: ScootalooConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TwitterConfig {
|
||||
pub username: String,
|
||||
pub consumer_key: String,
|
||||
pub consumer_secret: String,
|
||||
pub access_key: String,
|
||||
pub access_secret: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct MastodonConfig {
|
||||
pub base: String,
|
||||
pub client_id: String,
|
||||
pub client_secret: String,
|
||||
pub redirect: String,
|
||||
pub token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ScootalooConfig {
|
||||
pub db_path: String,
|
||||
pub cache_path: String,
|
||||
}
|
||||
|
||||
/// Parses the TOML file into a Config Struct
|
||||
pub fn parse_toml(toml_file: &str) -> Config {
|
||||
let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
|
||||
panic!("Cannot open config file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
|
||||
panic!("Cannot parse TOML file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
27
src/error.rs
Normal file
27
src/error.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ScootalooError {
|
||||
details: String,
|
||||
}
|
||||
|
||||
impl ScootalooError {
|
||||
pub fn new(msg: &str) -> ScootalooError {
|
||||
ScootalooError {
|
||||
details: String::from(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ScootalooError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "{}", self.details)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ScootalooError {
|
||||
fn description(&self) -> &str {
|
||||
&self.details
|
||||
}
|
||||
}
|
||||
|
||||
337
src/lib.rs
337
src/lib.rs
@@ -1,334 +1,53 @@
|
||||
// auto-imports
|
||||
mod error;
|
||||
use error::ScootalooError;
|
||||
|
||||
mod config;
|
||||
use config::Config;
|
||||
pub use config::parse_toml;
|
||||
|
||||
mod mastodon;
|
||||
use mastodon::{get_mastodon_token, build_basic_status};
|
||||
pub use mastodon::register;
|
||||
|
||||
mod twitter;
|
||||
use twitter::*;
|
||||
|
||||
mod util;
|
||||
|
||||
mod state;
|
||||
use state::{read_state, write_state};
|
||||
pub use state::init_db;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::HashMap,
|
||||
io::stdin,
|
||||
fmt,
|
||||
fs::{read_to_string, write},
|
||||
error::Error,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
|
||||
// toml
|
||||
use serde::Deserialize;
|
||||
|
||||
// egg-mode
|
||||
use egg_mode::{
|
||||
Token,
|
||||
KeyPair,
|
||||
entities::{UrlEntity, MediaEntity, MentionEntity, MediaType},
|
||||
user::UserID,
|
||||
tweet::{
|
||||
Tweet,
|
||||
user_timeline,
|
||||
},
|
||||
};
|
||||
// tokio
|
||||
use tokio::fs::remove_file;
|
||||
|
||||
// elefren
|
||||
use elefren::{
|
||||
prelude::*,
|
||||
apps::App,
|
||||
status_builder::StatusBuilder,
|
||||
scopes::Scopes,
|
||||
};
|
||||
|
||||
// reqwest
|
||||
use reqwest::Url;
|
||||
|
||||
// tokio
|
||||
use tokio::{
|
||||
io::copy,
|
||||
fs::{File, create_dir_all, remove_file},
|
||||
};
|
||||
|
||||
// htmlescape
|
||||
use htmlescape::decode_html;
|
||||
|
||||
// log
|
||||
use log::{info, warn, error, debug};
|
||||
|
||||
/**********
|
||||
* Generic usage functions
|
||||
***********/
|
||||
/*
|
||||
* Those functions are related to the Twitter side of things
|
||||
*/
|
||||
/// Reads last tweet id from a file
|
||||
fn read_state(s: &str) -> Option<u64> {
|
||||
let state = read_to_string(s);
|
||||
|
||||
if let Ok(s) = state {
|
||||
debug!("Last Tweet ID (from file): {}", &s);
|
||||
return s.parse::<u64>().ok();
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Writes last treated tweet id to a file
|
||||
fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
|
||||
write(f, format!("{}", s))
|
||||
}
|
||||
|
||||
/// Gets Twitter oauth2 token
|
||||
fn get_oauth2_token(config: &Config) -> Token {
|
||||
let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
|
||||
let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
|
||||
|
||||
Token::Access {
|
||||
consumer: con_token,
|
||||
access: access_token,
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets Twitter user timeline
|
||||
async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
|
||||
// fix the page size to 200 as it is the maximum Twitter authorizes
|
||||
let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
|
||||
.with_page_size(200)
|
||||
.older(lid)
|
||||
.await?;
|
||||
|
||||
Ok(feed.to_vec())
|
||||
}
|
||||
|
||||
/// Decodes urls from UrlEntities
|
||||
fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_urls = HashMap::new();
|
||||
|
||||
for url in urls {
|
||||
if url.expanded_url.is_some() {
|
||||
// unwrap is safe here as we just verified that there is something inside expanded_url
|
||||
decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
decoded_urls
|
||||
}
|
||||
|
||||
/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
|
||||
/// Fediverse
|
||||
fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_mentions = HashMap::new();
|
||||
|
||||
for um in ums {
|
||||
decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
|
||||
}
|
||||
|
||||
decoded_mentions
|
||||
}
|
||||
|
||||
/// Retrieves a single media from a tweet and store it in a temporary file
|
||||
async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
match m.media_type {
|
||||
MediaType::Photo => {
|
||||
return cache_media(&m.media_url_https, t).await;
|
||||
},
|
||||
_ => {
|
||||
match &m.video_info {
|
||||
Some(v) => {
|
||||
for variant in &v.variants {
|
||||
if variant.content_type == "video/mp4" {
|
||||
return cache_media(&variant.url, t).await;
|
||||
}
|
||||
}
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
|
||||
},
|
||||
None => {
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* Those functions are related to the Mastodon side of things
|
||||
*/
|
||||
/// Gets Mastodon Data
|
||||
fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
|
||||
let data = Data {
|
||||
base: Cow::from(String::from(&masto.base)),
|
||||
client_id: Cow::from(String::from(&masto.client_id)),
|
||||
client_secret: Cow::from(String::from(&masto.client_secret)),
|
||||
redirect: Cow::from(String::from(&masto.redirect)),
|
||||
token: Cow::from(String::from(&masto.token)),
|
||||
};
|
||||
|
||||
Mastodon::from(data)
|
||||
}
|
||||
|
||||
/// Builds toot text from tweet
|
||||
fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
|
||||
let mut toot = String::from(&tweet.text);
|
||||
|
||||
let decoded_urls = decode_urls(&tweet.entities.urls);
|
||||
|
||||
for decoded_url in decoded_urls {
|
||||
toot = toot.replace(&decoded_url.0, &decoded_url.1);
|
||||
}
|
||||
|
||||
let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
|
||||
|
||||
for decoded_mention in decoded_mentions {
|
||||
toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
|
||||
}
|
||||
|
||||
if let Ok(t) = decode_html(&toot) {
|
||||
toot = t;
|
||||
}
|
||||
|
||||
Ok(toot)
|
||||
}
|
||||
|
||||
/*
|
||||
* Generic private functions
|
||||
*/
|
||||
/// Gets and caches Twitter Media inside the determined temp dir
|
||||
async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
// create dir
|
||||
create_dir_all(t).await?;
|
||||
|
||||
// get file
|
||||
let mut response = reqwest::get(u).await?;
|
||||
|
||||
// create local file
|
||||
let url = Url::parse(u)?;
|
||||
let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
|
||||
.last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
|
||||
|
||||
let dest_filepath = format!("{}/{}", t, dest_filename);
|
||||
|
||||
let mut dest_file = File::create(&dest_filepath).await?;
|
||||
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
copy(&mut &*chunk, &mut dest_file).await?;
|
||||
}
|
||||
|
||||
Ok(dest_filepath)
|
||||
}
|
||||
|
||||
/**********
|
||||
* local error handler
|
||||
**********/
|
||||
#[derive(Debug)]
|
||||
struct ScootalooError {
|
||||
details: String,
|
||||
}
|
||||
|
||||
impl ScootalooError {
|
||||
fn new(msg: &str) -> ScootalooError {
|
||||
ScootalooError {
|
||||
details: String::from(msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ScootalooError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "{}", self.details)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ScootalooError {
|
||||
fn description(&self) -> &str {
|
||||
&self.details
|
||||
}
|
||||
}
|
||||
|
||||
/**********
|
||||
* Config structure
|
||||
***********/
|
||||
/// General configuration Struct
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
twitter: TwitterConfig,
|
||||
mastodon: MastodonConfig,
|
||||
scootaloo: ScootalooConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TwitterConfig {
|
||||
username: String,
|
||||
consumer_key: String,
|
||||
consumer_secret: String,
|
||||
access_key: String,
|
||||
access_secret: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct MastodonConfig {
|
||||
base: String,
|
||||
client_id: String,
|
||||
client_secret: String,
|
||||
redirect: String,
|
||||
token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ScootalooConfig {
|
||||
last_tweet_path: String,
|
||||
cache_path: String,
|
||||
}
|
||||
|
||||
/*********
|
||||
* Main functions
|
||||
*********/
|
||||
/// Parses the TOML file into a Config Struct
|
||||
pub fn parse_toml(toml_file: &str) -> Config {
|
||||
let toml_config = read_to_string(toml_file).unwrap_or_else(|e|
|
||||
panic!("Cannot open config file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
let config: Config = toml::from_str(&toml_config).unwrap_or_else(|e|
|
||||
panic!("Cannot parse TOML file {}: {}", toml_file, e)
|
||||
);
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
/// Generic register function
|
||||
/// As this function is supposed to be run only once, it will panic for every error it encounters
|
||||
/// Most of this function is a direct copy/paste of the official `elefren` crate
|
||||
pub fn register(host: &str) {
|
||||
let mut builder = App::builder();
|
||||
builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
|
||||
.redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
|
||||
.scopes(Scopes::write_all())
|
||||
.website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
|
||||
|
||||
let app = builder.build().expect("Cannot build the app");
|
||||
|
||||
let registration = Registration::new(host).register(app).expect("Cannot build registration object");
|
||||
let url = registration.authorize_url().expect("Cannot generate registration URI!");
|
||||
|
||||
println!("Click this link to authorize on Mastodon: {}", url);
|
||||
println!("Paste the returned authorization code: ");
|
||||
|
||||
let mut input = String::new();
|
||||
stdin().read_line(&mut input).expect("Unable to read back registration code!");
|
||||
|
||||
let code = input.trim();
|
||||
let mastodon = registration.complete(code).expect("Unable to create access token!");
|
||||
|
||||
let toml = toml::to_string(&*mastodon).unwrap();
|
||||
|
||||
println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
|
||||
}
|
||||
|
||||
/// This is where the magic happens
|
||||
#[tokio::main]
|
||||
pub async fn run(config: Config) {
|
||||
// retrieve the last tweet ID for the username
|
||||
let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
|
||||
let last_tweet_id = read_state(&config.scootaloo.db_path);
|
||||
|
||||
// get OAuth2 token
|
||||
let token = get_oauth2_token(&config);
|
||||
let token = get_oauth2_token(&config.twitter);
|
||||
|
||||
// get Mastodon instance
|
||||
let mastodon = get_mastodon_token(&config.mastodon);
|
||||
|
||||
// get user timeline feed (Vec<tweet>)
|
||||
let mut feed = get_user_timeline(&config, token, last_tweet_id)
|
||||
let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
|
||||
.await
|
||||
.unwrap_or_else(|e|
|
||||
panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
|
||||
@@ -411,7 +130,7 @@ pub async fn run(config: Config) {
|
||||
// last_tweet gathered not to be written
|
||||
|
||||
// write the current state (tweet ID) to avoid copying it another time
|
||||
write_state(&config.scootaloo.last_tweet_path, tweet.id).unwrap_or_else(|e|
|
||||
write_state(&config.scootaloo.db_path, tweet.id).unwrap_or_else(|e|
|
||||
panic!("Can’t write the last tweet retrieved: {}", e)
|
||||
);
|
||||
}
|
||||
|
||||
29
src/main.rs
29
src/main.rs
@@ -11,6 +11,8 @@ use simple_logger::SimpleLogger;
|
||||
// std
|
||||
use std::str::FromStr;
|
||||
|
||||
const DEFAULT_CONFIG_PATH: &'static str = "/usr/local/etc/scootaloo.toml";
|
||||
|
||||
fn main() {
|
||||
let matches = App::new(env!("CARGO_PKG_NAME"))
|
||||
.version(env!("CARGO_PKG_VERSION"))
|
||||
@@ -40,10 +42,29 @@ fn main() {
|
||||
.takes_value(true)
|
||||
.required(true)
|
||||
.display_order(1)))
|
||||
.subcommand(SubCommand::with_name("init")
|
||||
.version(env!("CARGO_PKG_VERSION"))
|
||||
.about("Command to init Scootaloo DB")
|
||||
.arg(Arg::with_name("config")
|
||||
.short("c")
|
||||
.long("config")
|
||||
.value_name("CONFIG_FILE")
|
||||
.help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml")
|
||||
.takes_value(true)
|
||||
.display_order(1)))
|
||||
.get_matches();
|
||||
if let Some(matches) = matches.subcommand_matches("register") {
|
||||
register(matches.value_of("host").unwrap());
|
||||
return;
|
||||
|
||||
match matches.subcommand() {
|
||||
("register", Some(sub_m)) => {
|
||||
register(sub_m.value_of("host").unwrap());
|
||||
return;
|
||||
},
|
||||
("init", Some(sub_m)) => {
|
||||
let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
|
||||
init_db(&config).unwrap();
|
||||
return;
|
||||
},
|
||||
_ => (),
|
||||
}
|
||||
|
||||
if matches.is_present("log_level") {
|
||||
@@ -56,7 +77,7 @@ fn main() {
|
||||
};
|
||||
}
|
||||
|
||||
let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
|
||||
let config = parse_toml(matches.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
|
||||
|
||||
run(config);
|
||||
}
|
||||
|
||||
106
src/mastodon.rs
Normal file
106
src/mastodon.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
// auto imports
|
||||
use crate::config::MastodonConfig;
|
||||
use crate::twitter::decode_urls;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
error::Error,
|
||||
collections::HashMap,
|
||||
io::stdin,
|
||||
};
|
||||
|
||||
// htmlescape
|
||||
use htmlescape::decode_html;
|
||||
|
||||
// egg-mode
|
||||
use egg_mode::{
|
||||
tweet::Tweet,
|
||||
entities::MentionEntity,
|
||||
};
|
||||
|
||||
// elefren
|
||||
use elefren::{
|
||||
prelude::*,
|
||||
apps::App,
|
||||
scopes::Scopes,
|
||||
};
|
||||
|
||||
|
||||
/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
|
||||
/// Fediverse
|
||||
fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_mentions = HashMap::new();
|
||||
|
||||
for um in ums {
|
||||
decoded_mentions.insert(format!("@{}", um.screen_name), format!("@{}@twitter.com", um.screen_name));
|
||||
}
|
||||
|
||||
decoded_mentions
|
||||
}
|
||||
|
||||
/// Gets Mastodon Data
|
||||
pub fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
|
||||
let data = Data {
|
||||
base: Cow::from(String::from(&masto.base)),
|
||||
client_id: Cow::from(String::from(&masto.client_id)),
|
||||
client_secret: Cow::from(String::from(&masto.client_secret)),
|
||||
redirect: Cow::from(String::from(&masto.redirect)),
|
||||
token: Cow::from(String::from(&masto.token)),
|
||||
};
|
||||
|
||||
Mastodon::from(data)
|
||||
}
|
||||
|
||||
/// Builds toot text from tweet
|
||||
pub fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
|
||||
let mut toot = String::from(&tweet.text);
|
||||
|
||||
let decoded_urls = decode_urls(&tweet.entities.urls);
|
||||
|
||||
for decoded_url in decoded_urls {
|
||||
toot = toot.replace(&decoded_url.0, &decoded_url.1);
|
||||
}
|
||||
|
||||
let decoded_mentions = twitter_mentions(&tweet.entities.user_mentions);
|
||||
|
||||
for decoded_mention in decoded_mentions {
|
||||
toot = toot.replace(&decoded_mention.0, &decoded_mention.1);
|
||||
}
|
||||
|
||||
if let Ok(t) = decode_html(&toot) {
|
||||
toot = t;
|
||||
}
|
||||
|
||||
Ok(toot)
|
||||
}
|
||||
|
||||
/// Generic register function
|
||||
/// As this function is supposed to be run only once, it will panic for every error it encounters
|
||||
/// Most of this function is a direct copy/paste of the official `elefren` crate
|
||||
pub fn register(host: &str) {
|
||||
let mut builder = App::builder();
|
||||
builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
|
||||
.redirect_uris(Cow::from(String::from("urn:ietf:wg:oauth:2.0:oob")))
|
||||
.scopes(Scopes::write_all())
|
||||
.website(Cow::from(String::from("https://framagit.org/veretcle/scootaloo")));
|
||||
|
||||
let app = builder.build().expect("Cannot build the app");
|
||||
|
||||
let registration = Registration::new(host).register(app).expect("Cannot build registration object");
|
||||
let url = registration.authorize_url().expect("Cannot generate registration URI!");
|
||||
|
||||
println!("Click this link to authorize on Mastodon: {}", url);
|
||||
println!("Paste the returned authorization code: ");
|
||||
|
||||
let mut input = String::new();
|
||||
stdin().read_line(&mut input).expect("Unable to read back registration code!");
|
||||
|
||||
let code = input.trim();
|
||||
let mastodon = registration.complete(code).expect("Unable to create access token!");
|
||||
|
||||
let toml = toml::to_string(&*mastodon).unwrap();
|
||||
|
||||
println!("Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", toml);
|
||||
}
|
||||
|
||||
38
src/state.rs
Normal file
38
src/state.rs
Normal file
@@ -0,0 +1,38 @@
|
||||
// auto-imports
|
||||
use crate::config::Config;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
fs::{read_to_string, write},
|
||||
error::Error,
|
||||
};
|
||||
|
||||
// log
|
||||
use log::debug;
|
||||
|
||||
/// Reads last tweet id from a file
|
||||
pub fn read_state(s: &str) -> Option<u64> {
|
||||
let state = read_to_string(s);
|
||||
|
||||
if let Ok(s) = state {
|
||||
debug!("Last Tweet ID (from file): {}", &s);
|
||||
return s.parse::<u64>().ok();
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Writes last treated tweet id to a file
|
||||
pub fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
|
||||
write(f, format!("{}", s))
|
||||
}
|
||||
|
||||
/*********
|
||||
* Main functions
|
||||
*********/
|
||||
/// Initiates the DB from path
|
||||
pub fn init_db(config: &Config) -> Result<(), Box<dyn Error>> {
|
||||
println!("config.scootaloo.db_path: {}", config.scootaloo.db_path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
83
src/twitter.rs
Normal file
83
src/twitter.rs
Normal file
@@ -0,0 +1,83 @@
|
||||
// auto-imports
|
||||
use crate::ScootalooError;
|
||||
use crate::config::TwitterConfig;
|
||||
use crate::util::cache_media;
|
||||
|
||||
// std
|
||||
use std::{
|
||||
error::Error,
|
||||
collections::HashMap,
|
||||
};
|
||||
|
||||
// egg-mode
|
||||
use egg_mode::{
|
||||
Token,
|
||||
KeyPair,
|
||||
entities::{UrlEntity, MediaEntity, MediaType},
|
||||
user::UserID,
|
||||
tweet::{
|
||||
Tweet,
|
||||
user_timeline,
|
||||
},
|
||||
};
|
||||
|
||||
/// Gets Twitter oauth2 token
|
||||
pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
|
||||
let con_token = KeyPair::new(String::from(&config.consumer_key), String::from(&config.consumer_secret));
|
||||
let access_token = KeyPair::new(String::from(&config.access_key), String::from(&config.access_secret));
|
||||
|
||||
Token::Access {
|
||||
consumer: con_token,
|
||||
access: access_token,
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets Twitter user timeline
|
||||
pub async fn get_user_timeline(config: &TwitterConfig, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
|
||||
// fix the page size to 200 as it is the maximum Twitter authorizes
|
||||
let (_, feed) = user_timeline(UserID::from(String::from(&config.username)), true, false, &token)
|
||||
.with_page_size(200)
|
||||
.older(lid)
|
||||
.await?;
|
||||
|
||||
Ok(feed.to_vec())
|
||||
}
|
||||
|
||||
/// Decodes urls from UrlEntities
|
||||
pub fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_urls = HashMap::new();
|
||||
|
||||
for url in urls {
|
||||
if url.expanded_url.is_some() {
|
||||
// unwrap is safe here as we just verified that there is something inside expanded_url
|
||||
decoded_urls.insert(String::from(&url.url), String::from(url.expanded_url.as_deref().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
decoded_urls
|
||||
}
|
||||
|
||||
/// Retrieves a single media from a tweet and store it in a temporary file
|
||||
pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
match m.media_type {
|
||||
MediaType::Photo => {
|
||||
return cache_media(&m.media_url_https, t).await;
|
||||
},
|
||||
_ => {
|
||||
match &m.video_info {
|
||||
Some(v) => {
|
||||
for variant in &v.variants {
|
||||
if variant.content_type == "video/mp4" {
|
||||
return cache_media(&variant.url, t).await;
|
||||
}
|
||||
}
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but no mp4 file URL is available", &m.url)).into());
|
||||
},
|
||||
None => {
|
||||
return Err(ScootalooError::new(&format!("Media Type for {} is video but does not contain any video_info", &m.url)).into());
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
37
src/util.rs
Normal file
37
src/util.rs
Normal file
@@ -0,0 +1,37 @@
|
||||
// std
|
||||
use std::error::Error;
|
||||
use crate::ScootalooError;
|
||||
|
||||
// reqwest
|
||||
use reqwest::Url;
|
||||
|
||||
// tokio
|
||||
use tokio::{
|
||||
io::copy,
|
||||
fs::{File, create_dir_all},
|
||||
};
|
||||
|
||||
/// Gets and caches Twitter Media inside the determined temp dir
|
||||
pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
// create dir
|
||||
create_dir_all(t).await?;
|
||||
|
||||
// get file
|
||||
let mut response = reqwest::get(u).await?;
|
||||
|
||||
// create local file
|
||||
let url = Url::parse(u)?;
|
||||
let dest_filename = url.path_segments().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?
|
||||
.last().ok_or_else(|| ScootalooError::new(&format!("Cannot determine the destination filename for {}", u)))?;
|
||||
|
||||
let dest_filepath = format!("{}/{}", t, dest_filename);
|
||||
|
||||
let mut dest_file = File::create(&dest_filepath).await?;
|
||||
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
copy(&mut &*chunk, &mut dest_file).await?;
|
||||
}
|
||||
|
||||
Ok(dest_filepath)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user