16 Commits

Author SHA1 Message Date
VC
31afb1cf7d Merge branch 'async_media_upload' into 'master'
Async media upload

See merge request veretcle/scootaloo!24
2022-11-08 13:35:06 +00:00
VC
4415c4ac12 refactor: better logic flow for uploading/deleting media 2022-11-08 10:54:42 +01:00
VC
89f1372f9f bump: version v0.8.0 2022-11-08 08:54:36 +01:00
VC
06904434c8 fix: indentation error when registering 2022-11-08 08:54:36 +01:00
VC
3c64df23bc refactor: add info/debug 2022-11-08 08:54:32 +01:00
VC
c62f67c3b3 refactor: simpler mtask var 2022-11-08 08:37:26 +01:00
VC
3b0e7234af refactor: downloads/uploads every media from a tweet async way 2022-11-08 08:37:17 +01:00
VC
62011b4b81 refactor: downloads/uploads every media from a tweet async way 2022-11-07 21:47:12 +01:00
VC
5ce3bde3e7 fix: remove unecessary \n in TOML conf 2022-11-07 18:25:55 +01:00
VC
ab4184c0ed Merge branch 'async_multi_account' into 'master'
feat: attempt for async treatment of all accounts

See merge request veretcle/scootaloo!23
2022-11-05 09:36:55 +00:00
VC
de758c7bda refactor: separate function for media ids 2022-11-05 10:23:21 +01:00
VC
df75520175 feat: async treatment of all accounts 2022-11-04 15:26:27 +01:00
VC
73244f9ecc Merge branch 'multi_account_scootaloo' into 'master'
Multi account scootaloo

See merge request veretcle/scootaloo!22
2022-11-03 22:38:26 +00:00
VC
dad49da090 feat: add multi-account ability 2022-11-03 23:30:50 +01:00
VC
44ec3edfe2 Merge branch 'rust_1_63' into 'master'
feat: adapt to rust 1.63

See merge request veretcle/scootaloo!21
2022-08-17 16:06:39 +00:00
VC
8673dd7866 feat: adapt to rust 1.63 2022-08-17 18:02:12 +02:00
12 changed files with 569 additions and 201 deletions

3
Cargo.lock generated
View File

@@ -2016,7 +2016,7 @@ dependencies = [
[[package]] [[package]]
name = "scootaloo" name = "scootaloo"
version = "0.6.0" version = "0.8.0"
dependencies = [ dependencies = [
"chrono", "chrono",
"clap", "clap",
@@ -2025,6 +2025,7 @@ dependencies = [
"futures 0.3.14", "futures 0.3.14",
"html-escape", "html-escape",
"log", "log",
"mime",
"reqwest 0.11.3", "reqwest 0.11.3",
"rusqlite", "rusqlite",
"serde", "serde",

View File

@@ -1,6 +1,6 @@
[package] [package]
name = "scootaloo" name = "scootaloo"
version = "0.6.0" version = "0.8.0"
authors = ["VC <veretcle+framagit@mateu.be>"] authors = ["VC <veretcle+framagit@mateu.be>"]
edition = "2021" edition = "2021"
@@ -20,6 +20,7 @@ html-escape = "^0.2"
reqwest = "^0.11" reqwest = "^0.11"
log = "^0.4" log = "^0.4"
simple_logger = "^2.1" simple_logger = "^2.1"
mime = "^0.3"
[profile.release] [profile.release]
strip = true strip = true

View File

@@ -10,23 +10,23 @@ If any of the last steps failed, the Toot gets published with the exact same tex
RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users. RT are excluded, replies are included when considered part of a thread (reply to self), not the actual replies to other Twitter users.
# Usage # Usage
## Configuring
First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this: First up, create a configuration file (default path is `/usr/local/etc/scootaloo.toml`). It will look like this:
```toml ```toml
[scootaloo] [scootaloo]
db_path = "/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable
db_path="/var/lib/scootaloo/scootaloo.sqlite" ## file containing the SQLite Tweet corresponding Toot DB, must be writeable cache_path = "/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
cache_path="/tmp/scootaloo" ## a dir where the temporary files will be download, must be writeable
[twitter] [twitter]
username="NintendojoFR" ## User Timeline to copy
## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps) ## Consumer/Access key for Twitter (can be generated at https://developer.twitter.com/en/apps)
consumer_key="MYCONSUMERKEY" consumer_key = "MYCONSUMERKEY"
consumer_secret="MYCONSUMERSECRET" consumer_secret = "MYCONSUMERSECRET"
access_key="MYACCESSKEY" access_key = "MYACCESSKEY"
access_secret="MYACCESSSECRET" access_secret = "MYACCESSSECRET"
[mastodon]
``` ```
Then run the command with the `init` subcommand to initiate the DB: Then run the command with the `init` subcommand to initiate the DB:
@@ -44,7 +44,8 @@ scootaloo register --host https://m.nintendojo.fr
This will give you the end of the TOML file. It will look like this: This will give you the end of the TOML file. It will look like this:
```toml ```toml
[mastodon] [mastodon.nintendojofr] ## account
twitter_screen_name="NintendojoFR" ## User Timeline to copy
base = "https://m.nintendojo.fr" base = "https://m.nintendojo.fr"
client_id = "MYCLIENTID" client_id = "MYCLIENTID"
client_secret = "MYCLIENTSECRET" client_secret = "MYCLIENTSECRET"
@@ -52,6 +53,10 @@ redirect = "urn:ietf:wg:oauth:2.0:oob"
token = "MYTOKEN" token = "MYTOKEN"
``` ```
You can add other account if you like, after the `[mastodon]` moniker. Scootaloo would theorically support an unlimited number of accounts.
## Running
You can then run the application via `cron` for example. Here is the generic usage: You can then run the application via `cron` for example. Here is the generic usage:
```sh ```sh
@@ -71,6 +76,7 @@ OPTIONS:
SUBCOMMANDS: SUBCOMMANDS:
help Prints this message or the help of the given subcommand(s) help Prints this message or the help of the given subcommand(s)
init Command to init Scootaloo DB init Command to init Scootaloo DB
migrate Command to migrate Scootaloo DB
register Command to register to a Mastodon Instance register Command to register to a Mastodon Instance
``` ```
@@ -86,5 +92,17 @@ sqlite3 /var/lib/scootaloo/scootaloo.sqlite
And inserting the data: And inserting the data:
```sql ```sql
INSERT INTO tweet_to_toot VALUES (1383782580412030982, ""); INSERT INTO tweet_to_toot VALUES ("<twitter_screen_name>", 1383782580412030982, "<twitter_screen_name>");
``` ```
The last value is supposed to be the Toot ID. It cannot be null, so you better initialize it with something unique, like the Twitter Screen Name for example.
# Migrating from Scootaloo ⩽ 0.6.1
The DB scheme has change between version 0.6.x and 0.7.x (this is due to the multi-account nature of Scootaloo from 0.7.x onward). You need to migrate your DB. You can do so by issuing the command:
```
scootaloo migrate
```
You can optionnally specify a screen name with the `--name` option. By default, itll take the first screen name in the config file.

View File

@@ -1,17 +1,17 @@
use std::{collections::HashMap, fs::read_to_string};
use serde::Deserialize; use serde::Deserialize;
use std::fs::read_to_string;
/// General configuration Struct /// General configuration Struct
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct Config { pub struct Config {
pub twitter: TwitterConfig, pub twitter: TwitterConfig,
pub mastodon: MastodonConfig, pub mastodon: HashMap<String, MastodonConfig>,
pub scootaloo: ScootalooConfig, pub scootaloo: ScootalooConfig,
} }
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct TwitterConfig { pub struct TwitterConfig {
pub username: String,
pub consumer_key: String, pub consumer_key: String,
pub consumer_secret: String, pub consumer_secret: String,
pub access_key: String, pub access_key: String,
@@ -20,6 +20,7 @@ pub struct TwitterConfig {
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct MastodonConfig { pub struct MastodonConfig {
pub twitter_screen_name: String,
pub base: String, pub base: String,
pub client_id: String, pub client_id: String,
pub client_secret: String, pub client_secret: String,

View File

@@ -13,45 +13,61 @@ mod twitter;
use twitter::*; use twitter::*;
mod util; mod util;
use crate::util::generate_media_ids;
mod state; mod state;
pub use state::init_db; pub use state::{init_db, migrate_db};
use state::{read_state, write_state, TweetToToot}; use state::{read_state, write_state, TweetToToot};
use elefren::{prelude::*, status_builder::StatusBuilder}; use elefren::{prelude::*, status_builder::StatusBuilder};
use log::{debug, error, info, warn}; use log::info;
use rusqlite::Connection; use rusqlite::Connection;
use std::borrow::Cow; use std::sync::Arc;
use tokio::fs::remove_file; use tokio::{spawn, sync::Mutex};
/// This is where the magic happens /// This is where the magic happens
#[tokio::main] #[tokio::main]
pub async fn run(config: Config) { pub async fn run(config: Config) {
// create the task vector for handling multiple accounts
let mut mtask = vec![];
// open the SQLite connection // open the SQLite connection
let conn = Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| { let conn = Arc::new(Mutex::new(
Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
panic!( panic!(
"Something went wrong when opening the DB {}: {}", "Something went wrong when opening the DB {}: {}",
&config.scootaloo.db_path, e &config.scootaloo.db_path, e
) )
}); }),
));
for mastodon_config in config.mastodon.into_values() {
// create temporary value for each task
let scootaloo_cache_path = config.scootaloo.cache_path.clone();
let token = get_oauth2_token(&config.twitter);
let task_conn = conn.clone();
let task = spawn(async move {
info!("Starting treating {}", &mastodon_config.twitter_screen_name);
// retrieve the last tweet ID for the username // retrieve the last tweet ID for the username
let last_tweet_id = read_state(&conn, None) let lconn = task_conn.lock().await;
let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)
.unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e)) .unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
.map(|s| s.tweet_id); .map(|s| s.tweet_id);
drop(lconn);
// get OAuth2 token
let token = get_oauth2_token(&config.twitter);
// get Mastodon instance // get Mastodon instance
let mastodon = get_mastodon_token(&config.mastodon); let mastodon = get_mastodon_token(&mastodon_config);
// get user timeline feed (Vec<tweet>) // get user timeline feed (Vec<tweet>)
let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id) let mut feed =
get_user_timeline(&mastodon_config.twitter_screen_name, &token, last_tweet_id)
.await .await
.unwrap_or_else(|e| { .unwrap_or_else(|e| {
panic!( panic!(
"Something went wrong when trying to retrieve {}s timeline: {}", "Something went wrong when trying to retrieve {}s timeline: {}",
&config.twitter.username, e &mastodon_config.twitter_screen_name, e
) )
}); });
@@ -65,67 +81,39 @@ pub async fn run(config: Config) {
feed.reverse(); feed.reverse();
for tweet in &feed { for tweet in &feed {
debug!("Treating Tweet {} inside feed", tweet.id); info!("Treating Tweet {} inside feed", tweet.id);
// initiate the toot_reply_id var // initiate the toot_reply_id var
let mut toot_reply_id: Option<String> = None; let mut toot_reply_id: Option<String> = None;
// determine if the tweet is part of a thread (response to self) or a standard response // determine if the tweet is part of a thread (response to self) or a standard response
if let Some(r) = &tweet.in_reply_to_screen_name { if let Some(r) = &tweet.in_reply_to_screen_name {
if r.to_lowercase() != config.twitter.username.to_lowercase() { if r.to_lowercase() != mastodon_config.twitter_screen_name.to_lowercase() {
// we are responding not threading // we are responding not threading
info!("Tweet is a direct response, skipping"); info!("Tweet is a direct response, skipping");
continue; continue;
} }
info!("Tweet is a thread"); info!("Tweet is a thread");
toot_reply_id = read_state(&conn, tweet.in_reply_to_status_id) // get the corresponding toot id
let lconn = task_conn.lock().await;
toot_reply_id = read_state(
&lconn,
&mastodon_config.twitter_screen_name,
tweet.in_reply_to_status_id,
)
.unwrap_or(None) .unwrap_or(None)
.map(|s| s.toot_id); .map(|s| s.toot_id);
drop(lconn);
}; };
// build basic status by just yielding text and dereferencing contained urls // build basic status by just yielding text and dereferencing contained urls
let mut status_text = build_basic_status(tweet); let mut status_text = build_basic_status(tweet);
let mut status_medias: Vec<String> = vec![]; // building associative media list
// reupload the attachments if any let (media_url, status_medias) =
if let Some(m) = &tweet.extended_entities { generate_media_ids(tweet, &scootaloo_cache_path, &mastodon).await;
for media in &m.media {
let local_tweet_media_path =
match get_tweet_media(media, &config.scootaloo.cache_path).await {
Ok(m) => m,
Err(e) => {
error!("Cannot get tweet media for {}: {}", &media.url, e);
continue;
}
};
let mastodon_media_ids = match mastodon status_text = status_text.replace(&media_url, "");
.media(Cow::from(local_tweet_media_path.to_owned()))
{
Ok(m) => {
remove_file(&local_tweet_media_path)
.await
.unwrap_or_else(|e|
warn!("Attachment for {} has been uploaded, but Im unable to remove the existing file: {}", &local_tweet_media_path, e)
);
m.id
}
Err(e) => {
error!(
"Attachment {} cannot be uploaded to Mastodon Instance: {}",
&local_tweet_media_path, e
);
continue;
}
};
status_medias.push(mastodon_media_ids); info!("Building corresponding Mastodon status");
// last step, removing the reference to the media from with the toots text
status_text = status_text.replace(&media.url, "");
}
}
// finished reuploading attachments, now lets do the toot baby!
debug!("Building corresponding Mastodon status");
let mut status_builder = StatusBuilder::new(); let mut status_builder = StatusBuilder::new();
@@ -135,6 +123,9 @@ pub async fn run(config: Config) {
status_builder.in_reply_to(&i); status_builder.in_reply_to(&i);
} }
// can be activated for test purposes
// status_builder.visibility(elefren::status_builder::Visibility::Private);
let status = status_builder let status = status_builder
.build() .build()
.unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text)); .unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
@@ -146,12 +137,25 @@ pub async fn run(config: Config) {
// last_tweet gathered not to be written // last_tweet gathered not to be written
let ttt_towrite = TweetToToot { let ttt_towrite = TweetToToot {
twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
tweet_id: tweet.id, tweet_id: tweet.id,
toot_id: published_status.id, toot_id: published_status.id,
}; };
// write the current state (tweet ID and toot ID) to avoid copying it another time // write the current state (tweet ID and toot ID) to avoid copying it another time
write_state(&conn, ttt_towrite) let lconn = task_conn.lock().await;
write_state(&lconn, ttt_towrite)
.unwrap_or_else(|e| panic!("Cant write the last tweet retrieved: {}", e)); .unwrap_or_else(|e| panic!("Cant write the last tweet retrieved: {}", e));
drop(lconn);
}
});
// push each task into the vec task
mtask.push(task);
}
// launch and wait for every handle
for handle in mtask {
handle.await.unwrap();
} }
} }

View File

@@ -43,7 +43,16 @@ fn main() {
.help("Base URL of the Mastodon instance to register to (no default)") .help("Base URL of the Mastodon instance to register to (no default)")
.takes_value(true) .takes_value(true)
.required(true) .required(true)
.display_order(1), .display_order(1)
)
.arg(
Arg::with_name("name")
.short("n")
.long("name")
.help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
.takes_value(true)
.required(true)
.display_order(2)
), ),
) )
.subcommand( .subcommand(
@@ -63,11 +72,36 @@ fn main() {
.display_order(1), .display_order(1),
), ),
) )
.subcommand(
SubCommand::with_name("migrate")
.version(env!("CARGO_PKG_VERSION"))
.about("Command to migrate Scootaloo DB")
.arg(
Arg::with_name("config")
.short("c")
.long("config")
.value_name("CONFIG_FILE")
.help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
.takes_value(true)
.display_order(1),
)
.arg(
Arg::with_name("name")
.short("n")
.long("name")
.help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
.takes_value(true)
.display_order(2)
)
)
.get_matches(); .get_matches();
match matches.subcommand() { match matches.subcommand() {
("register", Some(sub_m)) => { ("register", Some(sub_m)) => {
register(sub_m.value_of("host").unwrap()); register(
sub_m.value_of("host").unwrap(),
sub_m.value_of("name").unwrap(),
);
return; return;
} }
("init", Some(sub_m)) => { ("init", Some(sub_m)) => {
@@ -75,6 +109,17 @@ fn main() {
init_db(&config.scootaloo.db_path).unwrap(); init_db(&config.scootaloo.db_path).unwrap();
return; return;
} }
("migrate", Some(sub_m)) => {
let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
let config_twitter_screen_name =
&config.mastodon.values().next().unwrap().twitter_screen_name;
migrate_db(
&config.scootaloo.db_path,
sub_m.value_of("name").unwrap_or(config_twitter_screen_name),
)
.unwrap();
return;
}
_ => (), _ => (),
} }

View File

@@ -65,7 +65,7 @@ pub fn build_basic_status(tweet: &Tweet) -> String {
/// Generic register function /// Generic register function
/// As this function is supposed to be run only once, it will panic for every error it encounters /// As this function is supposed to be run only once, it will panic for every error it encounters
/// Most of this function is a direct copy/paste of the official `elefren` crate /// Most of this function is a direct copy/paste of the official `elefren` crate
pub fn register(host: &str) { pub fn register(host: &str, screen_name: &str) {
let mut builder = App::builder(); let mut builder = App::builder();
builder builder
.client_name(Cow::from(env!("CARGO_PKG_NAME").to_string())) .client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
@@ -100,7 +100,12 @@ pub fn register(host: &str) {
let toml = toml::to_string(&*mastodon).unwrap(); let toml = toml::to_string(&*mastodon).unwrap();
println!( println!(
"Please insert the following block at the end of your configuration file:\n[mastodon]\n{}", "Please insert the following block at the end of your configuration file:
[mastodon.{}]
twitter_screen_name = \"{}\"
{}",
screen_name.to_lowercase(),
screen_name,
toml toml
); );
} }

View File

@@ -1,10 +1,13 @@
use log::debug;
use rusqlite::{params, Connection, OptionalExtension};
use std::error::Error; use std::error::Error;
use log::debug;
use rusqlite::{params, Connection, OptionalExtension};
/// Struct for each query line /// Struct for each query line
#[derive(Debug)] #[derive(Debug)]
pub struct TweetToToot { pub struct TweetToToot {
pub twitter_screen_name: String,
pub tweet_id: u64, pub tweet_id: u64,
pub toot_id: String, pub toot_id: String,
} }
@@ -13,12 +16,13 @@ pub struct TweetToToot {
/// if a tweet_id is passed, read this particular tweet from DB /// if a tweet_id is passed, read this particular tweet from DB
pub fn read_state( pub fn read_state(
conn: &Connection, conn: &Connection,
n: &str,
s: Option<u64>, s: Option<u64>,
) -> Result<Option<TweetToToot>, Box<dyn Error>> { ) -> Result<Option<TweetToToot>, Box<dyn Error>> {
debug!("Reading tweet_id {:?}", s); debug!("Reading tweet_id {:?}", s);
let query: String = match s { let query: String = match s {
Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {}", i), Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {} and twitter_screen_name = \"{}\"", i, n),
None => "SELECT * FROM tweet_to_toot ORDER BY tweet_id DESC LIMIT 1".to_string(), None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{}\" ORDER BY tweet_id DESC LIMIT 1", n),
}; };
let mut stmt = conn.prepare(&query)?; let mut stmt = conn.prepare(&query)?;
@@ -26,8 +30,9 @@ pub fn read_state(
let t = stmt let t = stmt
.query_row([], |row| { .query_row([], |row| {
Ok(TweetToToot { Ok(TweetToToot {
tweet_id: row.get(0)?, twitter_screen_name: row.get("twitter_screen_name")?,
toot_id: row.get(1)?, tweet_id: row.get("tweet_id")?,
toot_id: row.get("toot_id")?,
}) })
}) })
.optional()?; .optional()?;
@@ -39,8 +44,8 @@ pub fn read_state(
pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> { pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
debug!("Write struct {:?}", t); debug!("Write struct {:?}", t);
conn.execute( conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)", "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id) VALUES (?1, ?2, ?3)",
params![t.tweet_id, t.toot_id], params![t.twitter_screen_name, t.tweet_id, t.toot_id],
)?; )?;
Ok(()) Ok(())
@@ -53,6 +58,7 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
conn.execute( conn.execute(
"CREATE TABLE IF NOT EXISTS tweet_to_toot ( "CREATE TABLE IF NOT EXISTS tweet_to_toot (
twitter_screen_name TEXT NOT NULL,
tweet_id INTEGER PRIMARY KEY, tweet_id INTEGER PRIMARY KEY,
toot_id TEXT UNIQUE toot_id TEXT UNIQUE
)", )",
@@ -62,6 +68,31 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
Ok(()) Ok(())
} }
/// Migrate DB from 0.6.x to 0.7.x
pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
debug!("Migrating DB for Scootaloo");
let conn = Connection::open(d)?;
let res = conn.execute(
&format!(
"ALTER TABLE tweet_to_toot
ADD COLUMN twitter_screen_name TEXT NOT NULL
DEFAULT \"{}\"",
s
),
[],
);
match res {
Err(e) => match e.to_string().as_str() {
"duplicate column name: twitter_screen_name" => Ok(()),
_ => Err(Box::new(e)),
},
_ => Ok(()),
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -93,9 +124,9 @@ mod tests {
let conn = Connection::open(d).unwrap(); let conn = Connection::open(d).unwrap();
conn.execute( conn.execute(
"INSERT INTO tweet_to_toot "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES VALUES
(100, 'A');", ('tamerelol', 100, 'A');",
[], [],
) )
.unwrap(); .unwrap();
@@ -114,6 +145,7 @@ mod tests {
let conn = Connection::open(d).unwrap(); let conn = Connection::open(d).unwrap();
let t_in = TweetToToot { let t_in = TweetToToot {
twitter_screen_name: "tamerelol".to_string(),
tweet_id: 123456789, tweet_id: 123456789,
toot_id: "987654321".to_string(), toot_id: "987654321".to_string(),
}; };
@@ -125,14 +157,16 @@ mod tests {
let t_out = stmt let t_out = stmt
.query_row([], |row| { .query_row([], |row| {
Ok(TweetToToot { Ok(TweetToToot {
tweet_id: row.get(0).unwrap(), twitter_screen_name: row.get("twitter_screen_name").unwrap(),
toot_id: row.get(1).unwrap(), tweet_id: row.get("tweet_id").unwrap(),
toot_id: row.get("toot_id").unwrap(),
}) })
}) })
.unwrap(); .unwrap();
assert_eq!(&t_out.twitter_screen_name, "tamerelol");
assert_eq!(t_out.tweet_id, 123456789); assert_eq!(t_out.tweet_id, 123456789);
assert_eq!(t_out.toot_id, "987654321".to_string()); assert_eq!(&t_out.toot_id, "987654321");
remove_file(d).unwrap(); remove_file(d).unwrap();
} }
@@ -146,15 +180,15 @@ mod tests {
let conn = Connection::open(d).unwrap(); let conn = Connection::open(d).unwrap();
conn.execute( conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id) "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES VALUES
(101, 'A'), ('tamerelol', 101, 'A'),
(102, 'B');", ('tamerelol', 102, 'B');",
[], [],
) )
.unwrap(); .unwrap();
let t_out = read_state(&conn, None).unwrap().unwrap(); let t_out = read_state(&conn, "tamerelol", None).unwrap().unwrap();
remove_file(d).unwrap(); remove_file(d).unwrap();
@@ -170,7 +204,7 @@ mod tests {
let conn = Connection::open(d).unwrap(); let conn = Connection::open(d).unwrap();
let t_out = read_state(&conn, None).unwrap(); let t_out = read_state(&conn, "tamerelol", None).unwrap();
remove_file(d).unwrap(); remove_file(d).unwrap();
@@ -186,14 +220,14 @@ mod tests {
let conn = Connection::open(d).unwrap(); let conn = Connection::open(d).unwrap();
conn.execute( conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id) "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES VALUES
(100, 'A');", ('tamerelol', 100, 'A');",
[], [],
) )
.unwrap(); .unwrap();
let t_out = read_state(&conn, Some(101)).unwrap(); let t_out = read_state(&conn, "tamerelol", Some(101)).unwrap();
remove_file(d).unwrap(); remove_file(d).unwrap();
@@ -209,18 +243,62 @@ mod tests {
let conn = Connection::open(d).unwrap(); let conn = Connection::open(d).unwrap();
conn.execute( conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id) "INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES VALUES
(100, 'A');", ('tamerelol', 100, 'A');",
[], [],
) )
.unwrap(); .unwrap();
let t_out = read_state(&conn, Some(100)).unwrap().unwrap(); let t_out = read_state(&conn, "tamerelol", Some(100)).unwrap().unwrap();
remove_file(d).unwrap(); remove_file(d).unwrap();
assert_eq!(t_out.tweet_id, 100); assert_eq!(t_out.tweet_id, 100);
assert_eq!(t_out.toot_id, "A"); assert_eq!(t_out.toot_id, "A");
} }
#[test]
fn test_migrate_db_add_column() {
let d = "/tmp/test_migrate_db_add_column.sqlite";
let conn = Connection::open(d).unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS tweet_to_toot (
tweet_id INTEGER PRIMARY KEY,
toot_id TEXT UNIQUE
)",
[],
)
.unwrap();
migrate_db(d, "tamerelol").unwrap();
let mut stmt = conn.prepare("PRAGMA table_info(tweet_to_toot);").unwrap();
let mut t = stmt.query([]).unwrap();
while let Some(row) = t.next().unwrap() {
if row.get::<usize, u8>(0).unwrap() == 2 {
assert_eq!(
row.get::<usize, String>(1).unwrap(),
"twitter_screen_name".to_string()
);
}
}
remove_file(d).unwrap();
}
#[test]
fn test_migrate_db_no_add_column() {
let d = "/tmp/test_migrate_db_no_add_column.sqlite";
init_db(d).unwrap();
migrate_db(d, "tamerelol").unwrap();
remove_file(d).unwrap();
}
} }

View File

@@ -29,17 +29,12 @@ pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
/// Gets Twitter user timeline /// Gets Twitter user timeline
pub async fn get_user_timeline( pub async fn get_user_timeline(
config: &TwitterConfig, screen_name: &str,
token: Token, token: &Token,
lid: Option<u64>, lid: Option<u64>,
) -> Result<Vec<Tweet>, Box<dyn Error>> { ) -> Result<Vec<Tweet>, Box<dyn Error>> {
// fix the page size to 200 as it is the maximum Twitter authorizes // fix the page size to 200 as it is the maximum Twitter authorizes
let (_, feed) = user_timeline( let (_, feed) = user_timeline(UserID::from(screen_name.to_owned()), true, false, token)
UserID::from(config.username.to_owned()),
true,
false,
&token,
)
.with_page_size(200) .with_page_size(200)
.older(lid) .older(lid)
.await?; .await?;
@@ -50,29 +45,137 @@ pub async fn get_user_timeline(
/// Retrieves a single media from a tweet and store it in a temporary file /// Retrieves a single media from a tweet and store it in a temporary file
pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> { pub async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
match m.media_type { match m.media_type {
MediaType::Photo => { MediaType::Photo => cache_media(&m.media_url_https, t).await,
return cache_media(&m.media_url_https, t).await;
}
_ => match &m.video_info { _ => match &m.video_info {
Some(v) => { Some(v) => match &v.variants.iter().find(|&x| x.content_type == "video/mp4") {
for variant in &v.variants { Some(u) => cache_media(&u.url, t).await,
if variant.content_type == "video/mp4" { None => Err(ScootalooError::new(&format!(
return cache_media(&variant.url, t).await;
}
}
return Err(ScootalooError::new(&format!(
"Media Type for {} is video but no mp4 file URL is available", "Media Type for {} is video but no mp4 file URL is available",
&m.url &m.url
)) ))
.into()); .into()),
} },
None => { None => Err(ScootalooError::new(&format!(
return Err(ScootalooError::new(&format!(
"Media Type for {} is video but does not contain any video_info", "Media Type for {} is video but does not contain any video_info",
&m.url &m.url
)) ))
.into()); .into()),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use egg_mode::entities::{
MediaSize, MediaSizes,
MediaType::{Gif, Photo},
ResizeMode::Crop,
ResizeMode::Fit,
VideoInfo, VideoVariant,
};
use std::fs::remove_dir_all;
const TMP_DIR: &'static str = "/tmp/scootaloo_get_tweet_media_test";
#[tokio::test]
async fn test_get_tweet_media() {
let m_photo = MediaEntity {
display_url: "pic.twitter.com/sHrwmP69Yv".to_string(),
expanded_url: "https://twitter.com/NintendojoFR/status/1555473821121056771/photo/1"
.to_string(),
id: 1555473771280080896,
range: (91, 114),
media_url: "http://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg".to_string(),
media_url_https: "https://pbs.twimg.com/media/FZYnJ1qWIAAReHt.jpg"
.to_string(),
sizes: MediaSizes {
thumb: MediaSize {
w: 150,
h: 150,
resize: Crop
},
small: MediaSize {
w: 680,
h: 510,
resize: Fit
},
medium: MediaSize {
w: 1200,
h: 900,
resize: Fit
},
large: MediaSize {
w: 1280,
h: 960,
resize: Fit
} }
}, },
source_status_id: None,
media_type: Photo,
url: "https://t.co/sHrwmP69Yv".to_string(),
video_info: None,
ext_alt_text: Some("Le menu «\u{a0}Classes » du jeu vidéo Xenoblade Chronicles 3 (Switch). Laffinité du personnage pour la classe est notée par quatre lettres : C, A, C, A (caca)."
.to_string())
}; };
let m_video = MediaEntity {
display_url: "pic.twitter.com/xDln0RrkjU".to_string(),
expanded_url: "https://twitter.com/NintendojoFR/status/1551822196833673218/photo/1"
.to_string(),
id: 1551822189711790081,
range: (275, 298),
media_url: "http://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg".to_string(),
media_url_https: "https://pbs.twimg.com/tweet_video_thumb/FYkuD0RXEAE-iDx.jpg"
.to_string(),
sizes: MediaSizes {
thumb: MediaSize {
w: 150,
h: 150,
resize: Crop,
},
small: MediaSize {
w: 320,
h: 240,
resize: Fit,
},
medium: MediaSize {
w: 320,
h: 240,
resize: Fit,
},
large: MediaSize {
w: 320,
h: 240,
resize: Fit,
},
},
source_status_id: None,
media_type: Gif,
url: "https://t.co/xDln0RrkjU".to_string(),
video_info: Some(VideoInfo {
aspect_ratio: (4, 3),
duration_millis: None,
variants: vec![VideoVariant {
bitrate: Some(0),
content_type: "video/mp4".parse::<mime::Mime>().unwrap(),
url: "https://video.twimg.com/tweet_video/FYkuD0RXEAE-iDx.mp4".to_string(),
}],
}),
ext_alt_text: Some("Scared Nintendo GIF".to_string()),
};
let tweet_media_photo = get_tweet_media(&m_photo, TMP_DIR).await.unwrap();
let tweet_media_video = get_tweet_media(&m_video, TMP_DIR).await.unwrap();
assert_eq!(
tweet_media_photo,
format!("{}/FZYnJ1qWIAAReHt.jpg", TMP_DIR)
);
assert_eq!(
tweet_media_video,
format!("{}/FYkuD0RXEAE-iDx.mp4", TMP_DIR)
);
remove_dir_all(TMP_DIR).unwrap();
}
} }

View File

@@ -1,11 +1,100 @@
use crate::ScootalooError; use crate::{twitter::get_tweet_media, ScootalooError};
use std::{borrow::Cow, error::Error};
use egg_mode::tweet::Tweet;
use elefren::prelude::*;
use log::{error, info, warn};
use reqwest::Url; use reqwest::Url;
use std::error::Error;
use tokio::{ use tokio::{
fs::{create_dir_all, File}, fs::{create_dir_all, remove_file, File},
io::copy, io::copy,
}; };
/// Generate associative table between media ids and tweet extended entities
pub async fn generate_media_ids(
tweet: &Tweet,
cache_path: &str,
mastodon: &Mastodon,
) -> (String, Vec<String>) {
let mut media_url = "".to_string();
let mut media_ids: Vec<String> = vec![];
if let Some(m) = &tweet.extended_entities {
// create tasks list
let mut tasks = vec![];
// size of media_ids vector, should be equal to the media vector
media_ids.resize(m.media.len(), String::new());
info!("{} medias in tweet", m.media.len());
for (i, media) in m.media.iter().enumerate() {
// attribute media url
media_url = media.url.clone();
// clone everything we need
let cache_path = String::from(cache_path);
let media = media.clone();
let mastodon = mastodon.clone();
let task = tokio::task::spawn(async move {
info!("Start treating {}", media.media_url_https);
// get the tweet embedded media
let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
Ok(l) => l,
Err(e) => {
return Err(ScootalooError::new(&format!(
"Cannot get tweet media for {}: {}",
&media.url, e
)))
}
};
// upload media to Mastodon
let mastodon_media = mastodon.media(Cow::from(local_tweet_media_path.to_owned()));
// at this point, we can safely erase the original file
// it doesnt matter if we cant remove, cache_media fn is idempotent
remove_file(&local_tweet_media_path).await.ok();
let mastodon_media = match mastodon_media {
Ok(m) => m,
Err(e) => {
return Err(ScootalooError::new(&format!(
"Attachment {} cannot be uploaded to Mastodon Instance: {}",
&local_tweet_media_path, e
)))
}
};
Ok((i, mastodon_media.id))
});
tasks.push(task);
}
for task in tasks {
match task.await {
// insert the media at the right place
Ok(Ok((i, v))) => media_ids[i] = v,
Ok(Err(e)) => warn!("{}", e),
Err(e) => error!("Something went wrong when joining the main thread: {}", e),
}
}
} else {
info!("No media in tweet");
}
// in case some media_ids slot remained empty due to errors, remove them
media_ids.retain(|x| !x.is_empty());
(media_url, media_ids)
}
/// Gets and caches Twitter Media inside the determined temp dir /// Gets and caches Twitter Media inside the determined temp dir
pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> { pub async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
// create dir // create dir

View File

@@ -10,20 +10,43 @@ fn test_parse_good_toml() {
); );
assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo"); assert_eq!(parse_good_toml.scootaloo.cache_path, "/tmp/scootaloo");
assert_eq!(parse_good_toml.twitter.username, "tamerelol");
assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key"); assert_eq!(parse_good_toml.twitter.consumer_key, "rand consumer key");
assert_eq!(parse_good_toml.twitter.consumer_secret, "secret"); assert_eq!(parse_good_toml.twitter.consumer_secret, "secret");
assert_eq!(parse_good_toml.twitter.access_key, "rand access key"); assert_eq!(parse_good_toml.twitter.access_key, "rand access key");
assert_eq!(parse_good_toml.twitter.access_secret, "super secret"); assert_eq!(parse_good_toml.twitter.access_secret, "super secret");
assert_eq!(parse_good_toml.mastodon.base, "https://m.nintendojo.fr");
assert_eq!(parse_good_toml.mastodon.client_id, "rand client id");
assert_eq!(parse_good_toml.mastodon.client_secret, "secret");
assert_eq!( assert_eq!(
parse_good_toml.mastodon.redirect, &parse_good_toml
.mastodon
.get("tamerelol")
.unwrap()
.twitter_screen_name,
"tamerelol"
);
assert_eq!(
&parse_good_toml.mastodon.get("tamerelol").unwrap().base,
"https://m.nintendojo.fr"
);
assert_eq!(
&parse_good_toml.mastodon.get("tamerelol").unwrap().client_id,
"rand client id"
);
assert_eq!(
&parse_good_toml
.mastodon
.get("tamerelol")
.unwrap()
.client_secret,
"secret"
);
assert_eq!(
&parse_good_toml.mastodon.get("tamerelol").unwrap().redirect,
"urn:ietf:wg:oauth:2.0:oob" "urn:ietf:wg:oauth:2.0:oob"
); );
assert_eq!(parse_good_toml.mastodon.token, "super secret"); assert_eq!(
&parse_good_toml.mastodon.get("tamerelol").unwrap().token,
"super secret"
);
} }
#[test] #[test]

View File

@@ -4,14 +4,14 @@ db_path="/var/random/scootaloo.sqlite"
cache_path="/tmp/scootaloo" cache_path="/tmp/scootaloo"
[twitter] [twitter]
username="tamerelol"
consumer_key="rand consumer key" consumer_key="rand consumer key"
consumer_secret="secret" consumer_secret="secret"
access_key="rand access key" access_key="rand access key"
access_secret="super secret" access_secret="super secret"
[mastodon] [mastodon]
[mastodon.tamerelol]
twitter_screen_name="tamerelol"
base = "https://m.nintendojo.fr" base = "https://m.nintendojo.fr"
client_id = "rand client id" client_id = "rand client id"
client_secret = "secret" client_secret = "secret"