dojo/scootaloo
1
0
Fork 0
mirror of https://framagit.org/veretcle/scootaloo.git synced 2025-07-20 17:11:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add multi-account ability

Browse Source
This commit is contained in:
VC
2022-11-03 16:14:25 +01:00
parent 44ec3edfe2
commit dad49da090
11 changed files with 341 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/config.rs
View File

@@ -1,17 +1,17 @@
use std::{collections::HashMap, fs::read_to_string};
use serde::Deserialize;
use std::fs::read_to_string;
/// General configuration Struct
#[derive(Debug, Deserialize)]
pub struct Config {
pub twitter: TwitterConfig,
pub mastodon: MastodonConfig,
pub mastodon: HashMap<String, MastodonConfig>,
pub scootaloo: ScootalooConfig,
}
#[derive(Debug, Deserialize)]
pub struct TwitterConfig {
pub username: String,
pub consumer_key: String,
pub consumer_secret: String,
pub access_key: String,
@@ -20,6 +20,7 @@ pub struct TwitterConfig {
#[derive(Debug, Deserialize)]
pub struct MastodonConfig {
pub twitter_screen_name: String,
pub base: String,
pub client_id: String,
pub client_secret: String,

217
src/lib.rs
View File

@@ -15,7 +15,7 @@ use twitter::*;
mod util;
mod state;
pub use state::init_db;
pub use state::{init_db, migrate_db};
use state::{read_state, write_state, TweetToToot};
use elefren::{prelude::*, status_builder::StatusBuilder};
@@ -27,131 +27,138 @@ use tokio::fs::remove_file;
/// This is where the magic happens
#[tokio::main]
pub async fn run(config: Config) {
// open the SQLite connection
let conn = Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
panic!(
"Something went wrong when opening the DB {}: {}",
&config.scootaloo.db_path, e
)
});
// retrieve the last tweet ID for the username
let last_tweet_id = read_state(&conn, None)
.unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
.map(|s| s.tweet_id);
// get OAuth2 token
let token = get_oauth2_token(&config.twitter);
// get Mastodon instance
let mastodon = get_mastodon_token(&config.mastodon);
// get user timeline feed (Vec<tweet>)
let mut feed = get_user_timeline(&config.twitter, token, last_tweet_id)
.await
.unwrap_or_else(|e| {
for mastodon_config in config.mastodon.values() {
// open the SQLite connection
let conn = Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
panic!(
"Something went wrong when trying to retrieve {}’s timeline: {}",
&config.twitter.username, e
"Something went wrong when opening the DB {}: {}",
&config.scootaloo.db_path, e
)
});
// retrieve the last tweet ID for the username
let last_tweet_id = read_state(&conn, &mastodon_config.twitter_screen_name, None)
.unwrap_or_else(|e| panic!("Cannot retrieve last_tweet_id: {}", e))
.map(|s| s.tweet_id);
// empty feed -> exiting
if feed.is_empty() {
info!("Nothing to retrieve since last time, exiting…");
return;
}
// get Mastodon instance
let mastodon = get_mastodon_token(mastodon_config);
// order needs to be chronological
feed.reverse();
// get user timeline feed (Vec<tweet>)
let mut feed = get_user_timeline(mastodon_config, &token, last_tweet_id)
.await
.unwrap_or_else(|e| {
panic!(
"Something went wrong when trying to retrieve {}’s timeline: {}",
&mastodon_config.twitter_screen_name, e
)
});
for tweet in &feed {
debug!("Treating Tweet {} inside feed", tweet.id);
// initiate the toot_reply_id var
let mut toot_reply_id: Option<String> = None;
// determine if the tweet is part of a thread (response to self) or a standard response
if let Some(r) = &tweet.in_reply_to_screen_name {
if r.to_lowercase() != config.twitter.username.to_lowercase() {
// we are responding not threading
info!("Tweet is a direct response, skipping");
continue;
}
info!("Tweet is a thread");
toot_reply_id = read_state(&conn, tweet.in_reply_to_status_id)
// empty feed -> exiting
if feed.is_empty() {
info!("Nothing to retrieve since last time, exiting…");
return;
}
// order needs to be chronological
feed.reverse();
for tweet in &feed {
debug!("Treating Tweet {} inside feed", tweet.id);
// initiate the toot_reply_id var
let mut toot_reply_id: Option<String> = None;
// determine if the tweet is part of a thread (response to self) or a standard response
if let Some(r) = &tweet.in_reply_to_screen_name {
if r.to_lowercase() != mastodon_config.twitter_screen_name.to_lowercase() {
// we are responding not threadin
info!("Tweet is a direct response, skipping");
continue;
}
info!("Tweet is a thread");
toot_reply_id = read_state(
&conn,
&mastodon_config.twitter_screen_name,
tweet.in_reply_to_status_id,
)
.unwrap_or(None)
.map(|s| s.toot_id);
};
};
// build basic status by just yielding text and dereferencing contained urls
let mut status_text = build_basic_status(tweet);
// build basic status by just yielding text and dereferencing contained urls
let mut status_text = build_basic_status(tweet);
let mut status_medias: Vec<String> = vec![];
// reupload the attachments if any
if let Some(m) = &tweet.extended_entities {
for media in &m.media {
let local_tweet_media_path =
match get_tweet_media(media, &config.scootaloo.cache_path).await {
Ok(m) => m,
Err(e) => {
error!("Cannot get tweet media for {}: {}", &media.url, e);
continue;
}
};
let mut status_medias: Vec<String> = vec![];
// reupload the attachments if any
if let Some(m) = &tweet.extended_entities {
for media in &m.media {
let local_tweet_media_path =
match get_tweet_media(media, &config.scootaloo.cache_path).await {
Ok(m) => m,
Err(e) => {
error!("Cannot get tweet media for {}: {}", &media.url, e);
continue;
}
};
let mastodon_media_ids = match mastodon
.media(Cow::from(local_tweet_media_path.to_owned()))
{
Ok(m) => {
remove_file(&local_tweet_media_path)
let mastodon_media_ids = match mastodon
.media(Cow::from(local_tweet_media_path.to_owned()))
{
Ok(m) => {
remove_file(&local_tweet_media_path)
.await
.unwrap_or_else(|e|
warn!("Attachment for {} has been uploaded, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
);
m.id
}
Err(e) => {
error!(
"Attachment {} cannot be uploaded to Mastodon Instance: {}",
&local_tweet_media_path, e
);
continue;
}
};
m.id
}
Err(e) => {
error!(
"Attachment {} cannot be uploaded to Mastodon Instance: {}",
&local_tweet_media_path, e
);
continue;
}
};
status_medias.push(mastodon_media_ids);
status_medias.push(mastodon_media_ids);
// last step, removing the reference to the media from with the toot’s text
status_text = status_text.replace(&media.url, "");
// last step, removing the reference to the media from with the toot’s text
status_text = status_text.replace(&media.url, "");
}
}
// finished reuploading attachments, now let’s do the toot baby!
debug!("Building corresponding Mastodon status");
let mut status_builder = StatusBuilder::new();
status_builder.status(&status_text).media_ids(status_medias);
if let Some(i) = toot_reply_id {
status_builder.in_reply_to(&i);
}
let status = status_builder
.build()
.unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
// publish status
// again unwrap is safe here as we are in the main thread
let published_status = mastodon.new_status(status).unwrap();
// this will panic if it cannot publish the status, which is a good thing, it allows the
// last_tweet gathered not to be written
let ttt_towrite = TweetToToot {
twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
tweet_id: tweet.id,
toot_id: published_status.id,
};
// write the current state (tweet ID and toot ID) to avoid copying it another time
write_state(&conn, ttt_towrite)
.unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e));
}
// finished reuploading attachments, now let’s do the toot baby!
debug!("Building corresponding Mastodon status");
let mut status_builder = StatusBuilder::new();
status_builder.status(&status_text).media_ids(status_medias);
if let Some(i) = toot_reply_id {
status_builder.in_reply_to(&i);
}
let status = status_builder
.build()
.unwrap_or_else(|_| panic!("Cannot build status with text {}", &status_text));
// publish status
// again unwrap is safe here as we are in the main thread
let published_status = mastodon.new_status(status).unwrap();
// this will panic if it cannot publish the status, which is a good thing, it allows the
// last_tweet gathered not to be written
let ttt_towrite = TweetToToot {
tweet_id: tweet.id,
toot_id: published_status.id,
};
// write the current state (tweet ID and toot ID) to avoid copying it another time
write_state(&conn, ttt_towrite)
.unwrap_or_else(|e| panic!("Can’t write the last tweet retrieved: {}", e));
}
}

49
src/main.rs
View File

@@ -43,7 +43,16 @@ fn main() {
.help("Base URL of the Mastodon instance to register to (no default)")
.takes_value(true)
.required(true)
.display_order(1),
.display_order(1)
)
.arg(
Arg::with_name("name")
.short("n")
.long("name")
.help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
.takes_value(true)
.required(true)
.display_order(2)
),
)
.subcommand(
@@ -63,11 +72,36 @@ fn main() {
.display_order(1),
),
)
.subcommand(
SubCommand::with_name("migrate")
.version(env!("CARGO_PKG_VERSION"))
.about("Command to migrate Scootaloo DB")
.arg(
Arg::with_name("config")
.short("c")
.long("config")
.value_name("CONFIG_FILE")
.help(&format!("TOML config file for scootaloo (default {})", DEFAULT_CONFIG_PATH))
.takes_value(true)
.display_order(1),
)
.arg(
Arg::with_name("name")
.short("n")
.long("name")
.help("Twitter Screen Name (like https://twitter.com/screen_name, no default)")
.takes_value(true)
.display_order(2)
)
)
.get_matches();
match matches.subcommand() {
("register", Some(sub_m)) => {
register(sub_m.value_of("host").unwrap());
register(
sub_m.value_of("host").unwrap(),
sub_m.value_of("name").unwrap(),
);
return;
}
("init", Some(sub_m)) => {
@@ -75,6 +109,17 @@ fn main() {
init_db(&config.scootaloo.db_path).unwrap();
return;
}
("migrate", Some(sub_m)) => {
let config = parse_toml(sub_m.value_of("config").unwrap_or(DEFAULT_CONFIG_PATH));
let config_twitter_screen_name =
&config.mastodon.values().next().unwrap().twitter_screen_name;
migrate_db(
&config.scootaloo.db_path,
sub_m.value_of("name").unwrap_or(config_twitter_screen_name),
)
.unwrap();
return;
}
_ => (),
}

9
src/mastodon.rs
View File

@@ -65,7 +65,7 @@ pub fn build_basic_status(tweet: &Tweet) -> String {
/// Generic register function
/// As this function is supposed to be run only once, it will panic for every error it encounters
/// Most of this function is a direct copy/paste of the official `elefren` crate
pub fn register(host: &str) {
pub fn register(host: &str, screen_name: &str) {
let mut builder = App::builder();
builder
.client_name(Cow::from(env!("CARGO_PKG_NAME").to_string()))
@@ -100,7 +100,12 @@ pub fn register(host: &str) {
let toml = toml::to_string(&*mastodon).unwrap();
println!(
"Please insert the following block at the end of your configuration file:\n[mastodon]\n{}",
"Please insert the following block at the end of your configuration file:
\n[mastodon.{}]
\ntwitter_screen_name = \"{}\"
\n{}",
screen_name.to_lowercase(),
screen_name,
toml
);
}

130
src/state.rs
View File

@@ -1,10 +1,13 @@
use log::debug;
use rusqlite::{params, Connection, OptionalExtension};
use std::error::Error;
use log::debug;
use rusqlite::{params, Connection, OptionalExtension};
/// Struct for each query line
#[derive(Debug)]
pub struct TweetToToot {
pub twitter_screen_name: String,
pub tweet_id: u64,
pub toot_id: String,
}
@@ -13,12 +16,13 @@ pub struct TweetToToot {
/// if a tweet_id is passed, read this particular tweet from DB
pub fn read_state(
conn: &Connection,
n: &str,
s: Option<u64>,
) -> Result<Option<TweetToToot>, Box<dyn Error>> {
debug!("Reading tweet_id {:?}", s);
let query: String = match s {
Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {}", i),
None => "SELECT * FROM tweet_to_toot ORDER BY tweet_id DESC LIMIT 1".to_string(),
Some(i) => format!("SELECT * FROM tweet_to_toot WHERE tweet_id = {} and twitter_screen_name = \"{}\"", i, n),
None => format!("SELECT * FROM tweet_to_toot WHERE twitter_screen_name = \"{}\" ORDER BY tweet_id DESC LIMIT 1", n),
};
let mut stmt = conn.prepare(&query)?;
@@ -26,8 +30,9 @@ pub fn read_state(
let t = stmt
.query_row([], |row| {
Ok(TweetToToot {
tweet_id: row.get(0)?,
toot_id: row.get(1)?,
twitter_screen_name: row.get("twitter_screen_name")?,
tweet_id: row.get("tweet_id")?,
toot_id: row.get("toot_id")?,
})
})
.optional()?;
@@ -39,8 +44,8 @@ pub fn read_state(
pub fn write_state(conn: &Connection, t: TweetToToot) -> Result<(), Box<dyn Error>> {
debug!("Write struct {:?}", t);
conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id) VALUES (?1, ?2)",
params![t.tweet_id, t.toot_id],
"INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id) VALUES (?1, ?2, ?3)",
params![t.twitter_screen_name, t.tweet_id, t.toot_id],
)?;
Ok(())
@@ -53,8 +58,9 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
conn.execute(
"CREATE TABLE IF NOT EXISTS tweet_to_toot (
tweet_id INTEGER PRIMARY KEY,
toot_id TEXT UNIQUE
twitter_screen_name TEXT NOT NULL,
tweet_id INTEGER PRIMARY KEY,
toot_id TEXT UNIQUE
)",
[],
)?;
@@ -62,6 +68,31 @@ pub fn init_db(d: &str) -> Result<(), Box<dyn Error>> {
Ok(())
}
/// Migrate DB from 0.6.x to 0.7.x
pub fn migrate_db(d: &str, s: &str) -> Result<(), Box<dyn Error>> {
debug!("Migrating DB for Scootaloo");
let conn = Connection::open(d)?;
let res = conn.execute(
&format!(
"ALTER TABLE tweet_to_toot
ADD COLUMN twitter_screen_name TEXT NOT NULL
DEFAULT \"{}\"",
s
),
[],
);
match res {
Err(e) => match e.to_string().as_str() {
"duplicate column name: twitter_screen_name" => Ok(()),
_ => Err(Box::new(e)),
},
_ => Ok(()),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -93,9 +124,9 @@ mod tests {
let conn = Connection::open(d).unwrap();
conn.execute(
"INSERT INTO tweet_to_toot
"INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES
(100, 'A');",
('tamerelol', 100, 'A');",
[],
)
.unwrap();
@@ -114,6 +145,7 @@ mod tests {
let conn = Connection::open(d).unwrap();
let t_in = TweetToToot {
twitter_screen_name: "tamerelol".to_string(),
tweet_id: 123456789,
toot_id: "987654321".to_string(),
};
@@ -125,14 +157,16 @@ mod tests {
let t_out = stmt
.query_row([], |row| {
Ok(TweetToToot {
tweet_id: row.get(0).unwrap(),
toot_id: row.get(1).unwrap(),
twitter_screen_name: row.get("twitter_screen_name").unwrap(),
tweet_id: row.get("tweet_id").unwrap(),
toot_id: row.get("toot_id").unwrap(),
})
})
.unwrap();
assert_eq!(&t_out.twitter_screen_name, "tamerelol");
assert_eq!(t_out.tweet_id, 123456789);
assert_eq!(t_out.toot_id, "987654321".to_string());
assert_eq!(&t_out.toot_id, "987654321");
remove_file(d).unwrap();
}
@@ -146,15 +180,15 @@ mod tests {
let conn = Connection::open(d).unwrap();
conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id)
"INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES
(101, 'A'),
(102, 'B');",
('tamerelol', 101, 'A'),
('tamerelol', 102, 'B');",
[],
)
.unwrap();
let t_out = read_state(&conn, None).unwrap().unwrap();
let t_out = read_state(&conn, "tamerelol", None).unwrap().unwrap();
remove_file(d).unwrap();
@@ -170,7 +204,7 @@ mod tests {
let conn = Connection::open(d).unwrap();
let t_out = read_state(&conn, None).unwrap();
let t_out = read_state(&conn, "tamerelol", None).unwrap();
remove_file(d).unwrap();
@@ -186,14 +220,14 @@ mod tests {
let conn = Connection::open(d).unwrap();
conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id)
"INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES
(100, 'A');",
('tamerelol', 100, 'A');",
[],
)
.unwrap();
let t_out = read_state(&conn, Some(101)).unwrap();
let t_out = read_state(&conn, "tamerelol", Some(101)).unwrap();
remove_file(d).unwrap();
@@ -209,18 +243,62 @@ mod tests {
let conn = Connection::open(d).unwrap();
conn.execute(
"INSERT INTO tweet_to_toot (tweet_id, toot_id)
"INSERT INTO tweet_to_toot (twitter_screen_name, tweet_id, toot_id)
VALUES
(100, 'A');",
('tamerelol', 100, 'A');",
[],
)
.unwrap();
let t_out = read_state(&conn, Some(100)).unwrap().unwrap();
let t_out = read_state(&conn, "tamerelol", Some(100)).unwrap().unwrap();
remove_file(d).unwrap();
assert_eq!(t_out.tweet_id, 100);
assert_eq!(t_out.toot_id, "A");
}
#[test]
fn test_migrate_db_add_column() {
let d = "/tmp/test_migrate_db_add_column.sqlite";
let conn = Connection::open(d).unwrap();
conn.execute(
"CREATE TABLE IF NOT EXISTS tweet_to_toot (
tweet_id INTEGER PRIMARY KEY,
toot_id TEXT UNIQUE
)",
[],
)
.unwrap();
migrate_db(d, "tamerelol").unwrap();
let mut stmt = conn.prepare("PRAGMA table_info(tweet_to_toot);").unwrap();
let mut t = stmt.query([]).unwrap();
while let Some(row) = t.next().unwrap() {
if row.get::<usize, u8>(0).unwrap() == 2 {
assert_eq!(
row.get::<usize, String>(1).unwrap(),
"twitter_screen_name".to_string()
);
}
}
remove_file(d).unwrap();
}
#[test]
fn test_migrate_db_no_add_column() {
let d = "/tmp/test_migrate_db_no_add_column.sqlite";
init_db(d).unwrap();
migrate_db(d, "tamerelol").unwrap();
remove_file(d).unwrap();
}
}

9
src/twitter.rs
View File

@@ -1,3 +1,4 @@
use crate::config::MastodonConfig;
use crate::config::TwitterConfig;
use crate::util::cache_media;
use crate::ScootalooError;
@@ -29,16 +30,16 @@ pub fn get_oauth2_token(config: &TwitterConfig) -> Token {
/// Gets Twitter user timeline
pub async fn get_user_timeline(
config: &TwitterConfig,
token: Token,
config: &MastodonConfig,
token: &Token,
lid: Option<u64>,
) -> Result<Vec<Tweet>, Box<dyn Error>> {
// fix the page size to 200 as it is the maximum Twitter authorizes
let (_, feed) = user_timeline(
UserID::from(config.username.to_owned()),
UserID::from(config.twitter_screen_name.to_owned()),
true,
false,
&token,
token,
)
.with_page_size(200)
.older(lid)