From 48b8eaaa5beeafbb4f95a208f1e51ef65c267d20 Mon Sep 17 00:00:00 2001 From: VC Date: Sat, 23 Apr 2022 10:01:00 +0200 Subject: [PATCH] feature: state is held into a sqlite db --- README.md | 19 ++++---- config.rs | 35 -------------- src/lib.rs | 21 ++++++-- src/state.rs | 133 +++++++++++++++++++++++++++++++++++---------------- 4 files changed, 120 insertions(+), 88 deletions(-) delete mode 100644 config.rs diff --git a/README.md b/README.md index f6f53ed..0a17168 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ access_secret="MYACCESSSECRET" ``` Then run the command with the `init` subcommand to initiate the DB: -``` +```sh scootaloo init ``` @@ -52,7 +52,9 @@ token = "MYTOKEN" You can then run the application via `cron` for example. Here is the generic usage: -``` +```sh +A Twitter to Mastodon bot + USAGE: scootaloo [OPTIONS] [SUBCOMMAND] @@ -62,20 +64,21 @@ FLAGS: OPTIONS: -c, --config TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml) + -l, --loglevel Log level. Valid values are: Off, Warn, Error, Info, Debug SUBCOMMANDS: help Prints this message or the help of the given subcommand(s) + init Command to init Scootaloo DB register Command to register to a Mastodon Instance ``` # Quirks -Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the `last_tweet` file before copying an old account. +Scootaloo does not respect the spam limits imposed by Mastodon: it will make a 429 error if too much Tweets are converted to Toots in a short amount of time (and it will not recover from it). By default, it gets the last 200 tweets from the user timeline (which is a lot!). It is recommended to put a Tweet number into the DB file before copying an old account. -You can do that with a command like: +You can can insert it like this: ```sh -echo -n '8189881949849' > last_tweet +sqlite3 /var/lib/scootaloo/scootaloo.sqlite +INSERT INTO tweet_to_toot VALUES (1383782580412030982, ""); +.quit ``` - -**This file should only contain the last tweet ID without any other char (no EOL or new line).** - diff --git a/config.rs b/config.rs deleted file mode 100644 index 0fa7025..0000000 --- a/config.rs +++ /dev/null @@ -1,35 +0,0 @@ -use scootaloo::parse_toml; - -#[test] -fn parse_good_toml() { - let tConfig = TwitterConfig { - username: "test", - consumer_key: "foo", - consumer_secret: "bar", - access_key: "secret", - access_secret: "super secret", - }; - - let mConfig = MastodonConfig { - base: "https://www.example.com", - client_id: "my_id", - client_secret: "this is secret", - redirect: "ooo:oooo:o", - token: "super secret", - }; - - let sConfig = ScootalooConfig { - db_path: "/tmp/scootaloo/scootaloo.db", - cache_path: "/tmp", - }; - - let test_config = Config { - twitter: tConfig, - mastodon: mConfig, - scootaloo: sConfig, - }; - - let parsed_config = parse_toml("tests/right_config.toml"); - - assert_eq!(parsed_config, test_config); -} diff --git a/src/lib.rs b/src/lib.rs index 643ffd5..f7834f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ use twitter::*; mod util; mod state; -use state::{read_state, write_state}; +use state::{read_state, write_state, TweetToToot}; pub use state::init_db; // std @@ -34,11 +34,19 @@ use elefren::{ // log use log::{info, warn, error, debug}; +// rusqlite +use rusqlite::Connection; + /// This is where the magic happens #[tokio::main] pub async fn run(config: Config) { + // open the SQLite connection + let conn = Connection::open(&config.scootaloo.db_path).unwrap(); // retrieve the last tweet ID for the username - let last_tweet_id = read_state(&config.scootaloo.db_path); + let last_tweet_id = match read_state(&conn, None).unwrap() { + Some(i) => Some(i.tweet_id), + None => None, + }; // get OAuth2 token let token = get_oauth2_token(&config.twitter); @@ -125,12 +133,17 @@ pub async fn run(config: Config) { // publish status // again unwrap is safe here as we are in the main thread - mastodon.new_status(status).unwrap(); + let published_status = mastodon.new_status(status).unwrap(); // this will panic if it cannot publish the status, which is a good thing, it allows the // last_tweet gathered not to be written + let ttt_towrite = TweetToToot { + tweet_id: tweet.id, + toot_id: published_status.id, + }; + // write the current state (tweet ID) to avoid copying it another time - write_state(&config.scootaloo.db_path, tweet.id).unwrap_or_else(|e| + write_state(&conn, ttt_towrite).unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e) ); } diff --git a/src/state.rs b/src/state.rs index 42b2f16..d4e91f6 100644 --- a/src/state.rs +++ b/src/state.rs @@ -2,32 +2,52 @@ use crate::config::ScootalooConfig; // std -use std::{ - fs::{read_to_string, write}, - error::Error, -}; +use std::error::Error; // log use log::debug; // rusqlite -use rusqlite::{Connection, OpenFlags, params}; +use rusqlite::{Connection, params, OptionalExtension}; -/// Reads last tweet id from a file -pub fn read_state(s: &str) -> Option { - let state = read_to_string(s); - - if let Ok(s) = state { - debug!("Last Tweet ID (from file): {}", &s); - return s.parse::().ok(); - } - - None +/// Struct for each query line +#[derive(Debug)] +pub struct TweetToToot { + pub tweet_id: u64, + pub toot_id: String, } -/// Writes last treated tweet id to a file -pub fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> { - write(f, format!("{}", s)) +/// if None is passed, read the last tweet from DB +/// if a tweet_id is passed, read this particular tweet from DB +pub fn read_state(conn: &Connection, s: Option) -> Result, Box> { + debug!("Reading tweet_id {:?}", s); + let query: String; + match s { + Some(i) => query = format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {}", i), + None => query = String::from("SELECT * FROM tweet_to_toot ORDER BY tweet_id DESC LIMIT 1"), + }; + + let mut stmt = conn.prepare(&query)?; + + let t = stmt.query_row([], |row| { + Ok(TweetToToot { + tweet_id: row.get(0)?, + toot_id: row.get(1)?, + }) + }).optional()?; + + Ok(t) +} + +/// Writes last treated tweet id and toot id to the db +pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box> { + debug!("Write struct {:?}", t); + conn.execute( + "INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)", + params![t.tweet_id, t.toot_id], + )?; + + Ok(()) } /********* @@ -35,6 +55,7 @@ pub fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> { *********/ /// Initiates the DB from path pub fn init_db(config: &ScootalooConfig) -> Result<(), Box> { + debug!("Initializing DB for Scootaloo"); let conn = Connection::open(&config.db_path)?; conn.execute( @@ -57,7 +78,7 @@ mod tests { }; #[test] - fn test_init_db() { + fn test_db() { let scootaloo_config = ScootalooConfig { db_path: String::from("/tmp/test_init_db.sqlite"), cache_path: String::from("/tmp/scootaloo"), @@ -69,33 +90,63 @@ mod tests { assert!(Path::new(&scootaloo_config.db_path).exists()); // open said file - let conn = Connection::open_with_flags(&scootaloo_config.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); - + let conn = Connection::open(&scootaloo_config.db_path).unwrap(); conn.execute( "SELECT * from tweet_to_toot;", [], ).unwrap(); - conn.close().unwrap(); + // write a state to DB + let t = TweetToToot { + tweet_id: 123456789, + toot_id: String::from("987654321"), + }; + write_state(&conn, t).unwrap(); + + let mut stmt = conn.prepare("SELECT * FROM tweet_to_toot limit 1;").unwrap(); + let mut rows = stmt.query([]).unwrap(); + + while let Some(row) = rows.next().unwrap() { + assert_eq!(123456789 as u64, row.get::<_, u64>(0).unwrap()); + assert_eq!("987654321", row.get::<_, String>(1).unwrap()); + } + + // write several other states + let (t1, t2) = ( + TweetToToot { + tweet_id: 11111111, + toot_id: String::from("tamerelol"), + }, + TweetToToot { + tweet_id: 1123456789, + toot_id: String::from("tonperemdr"), + }); + + write_state(&conn, t1).unwrap(); + write_state(&conn, t2).unwrap(); + + match read_state(&conn, None).unwrap() { + Some(i) => { + assert_eq!(1123456789, i.tweet_id); + assert_eq!("tonperemdr", &i.toot_id); + }, + None => panic!("This should not happen!"), + } + + match read_state(&conn, Some(11111111)).unwrap() { + Some(i) => { + assert_eq!(11111111, i.tweet_id); + assert_eq!("tamerelol", &i.toot_id); + }, + None => panic!("This should not happen!"), + } + + match read_state(&conn, Some(0000000)).unwrap() { + Some(_) => panic!("This should not happen"), + _ => (), + } + remove_file(&scootaloo_config.db_path).unwrap(); } - - #[test] - fn test_read_state() { - let scootaloo_config = ScootalooConfig { - db_path: String::from("/tmp/test_read_state.sqlite"), - cache_path: String::from("/tmp/scootaloo"), - }; - - init_db(&scootaloo_config).unwrap(); - - let conn = Connection::open(&scootaloo_config.db_path).unwrap(); - - conn.execute( - "INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)", - params![123456789 as u64, String::from("987654321")], - ).unwrap(); - - - } } +