mirror of
https://framagit.org/veretcle/scootaloo.git
synced 2025-07-20 17:11:19 +02:00
313 lines
12 KiB
Rust
313 lines
12 KiB
Rust
mod error;
|
||
use error::ScootalooError;
|
||
|
||
mod config;
|
||
pub use config::parse_toml;
|
||
use config::Config;
|
||
|
||
mod mastodon;
|
||
pub use mastodon::register;
|
||
use mastodon::*;
|
||
|
||
mod twitter;
|
||
use twitter::*;
|
||
|
||
mod util;
|
||
use util::{base64_media, generate_media_ids};
|
||
|
||
mod state;
|
||
pub use state::{init_db, migrate_db};
|
||
use state::{read_state, write_state, TweetToToot};
|
||
|
||
use futures::StreamExt;
|
||
use html_escape::decode_html_entities;
|
||
use log::info;
|
||
use megalodon::{
|
||
megalodon::PostStatusInputOptions, megalodon::UpdateCredentialsInputOptions, Megalodon,
|
||
};
|
||
use regex::Regex;
|
||
use rusqlite::Connection;
|
||
use std::sync::Arc;
|
||
use tokio::{spawn, sync::Mutex};
|
||
|
||
const DEFAULT_RATE_LIMIT: usize = 4;
|
||
const DEFAULT_PAGE_SIZE: i32 = 200;
|
||
|
||
/// This is where the magic happens
|
||
#[tokio::main]
|
||
pub async fn run(config: Config) {
|
||
// open the SQLite connection
|
||
let conn = Arc::new(Mutex::new(
|
||
Connection::open(&config.scootaloo.db_path).unwrap_or_else(|e| {
|
||
panic!(
|
||
"Something went wrong when opening the DB {}: {}",
|
||
&config.scootaloo.db_path, e
|
||
)
|
||
}),
|
||
));
|
||
|
||
let global_mastodon_config = Arc::new(Mutex::new(config.mastodon.clone()));
|
||
|
||
let display_url_re = config
|
||
.scootaloo
|
||
.show_url_as_display_url_for
|
||
.as_ref()
|
||
.map(|r|
|
||
// we want to panic in case the RE is not valid
|
||
Regex::new(r).unwrap());
|
||
|
||
let mut stream = futures::stream::iter(config.mastodon.into_values())
|
||
.map(|mastodon_config| {
|
||
// calculate Twitter page size
|
||
let page_size = mastodon_config
|
||
.twitter_page_size
|
||
.unwrap_or_else(|| config.twitter.page_size.unwrap_or(DEFAULT_PAGE_SIZE));
|
||
|
||
// create temporary value for each task
|
||
let scootaloo_cache_path = config.scootaloo.cache_path.clone();
|
||
let scootaloo_alt_services = config.scootaloo.alternative_services_for.clone();
|
||
let display_url_re = display_url_re.clone();
|
||
let token = get_oauth2_token(&config.twitter);
|
||
let task_conn = conn.clone();
|
||
let global_mastodon_config = global_mastodon_config.clone();
|
||
|
||
spawn(async move {
|
||
info!("Starting treating {}", &mastodon_config.twitter_screen_name);
|
||
// retrieve the last tweet ID for the username
|
||
let lconn = task_conn.lock().await;
|
||
let last_tweet_id = read_state(&lconn, &mastodon_config.twitter_screen_name, None)?
|
||
.map(|r| r.tweet_id);
|
||
drop(lconn);
|
||
|
||
// get reversed, curated user timeline
|
||
let feed = get_user_timeline(
|
||
&mastodon_config.twitter_screen_name,
|
||
&token,
|
||
last_tweet_id,
|
||
page_size,
|
||
)
|
||
.await?;
|
||
|
||
// get Mastodon instance
|
||
let mastodon = get_mastodon_token(&mastodon_config);
|
||
|
||
for tweet in &feed {
|
||
info!("Treating Tweet {} inside feed", tweet.id);
|
||
|
||
// basic toot text
|
||
let mut status_text = tweet.text.clone();
|
||
|
||
// add mentions and smart mentions
|
||
if !&tweet.entities.user_mentions.is_empty() {
|
||
info!("Tweet contains mentions, add them!");
|
||
let global_mastodon_config = global_mastodon_config.lock().await;
|
||
twitter_mentions(
|
||
&mut status_text,
|
||
&tweet.entities.user_mentions,
|
||
&global_mastodon_config,
|
||
);
|
||
drop(global_mastodon_config);
|
||
}
|
||
|
||
if !&tweet.entities.urls.is_empty() {
|
||
info!("Tweet contains links, add them!");
|
||
let mut associated_urls =
|
||
associate_urls(&tweet.entities.urls, &display_url_re);
|
||
|
||
if let Some(q) = &tweet.quoted_status {
|
||
if let Some(u) = &q.user {
|
||
info!(
|
||
"Tweet {} contains a quote, we try to find it within the DB",
|
||
tweet.id
|
||
);
|
||
// we know we have a quote and a user, we can lock both the
|
||
// connection to DB and global_config
|
||
// we will release them manually as soon as they’re useless
|
||
let lconn = task_conn.lock().await;
|
||
let global_mastodon_config = global_mastodon_config.lock().await;
|
||
if let Ok(Some(r)) = read_state(&lconn, &u.screen_name, Some(q.id))
|
||
{
|
||
info!("We have found the associated toot({})", &r.toot_id);
|
||
// drop conn immediately after the request: we won’t need it
|
||
// any more and the treatment there might be time-consuming
|
||
drop(lconn);
|
||
if let Some((m, t)) =
|
||
find_mastodon_screen_name_by_twitter_screen_name(
|
||
&r.twitter_screen_name,
|
||
&global_mastodon_config,
|
||
)
|
||
{
|
||
// drop the global conf, we have all we required, no need
|
||
// to block it further
|
||
drop(global_mastodon_config);
|
||
replace_tweet_by_toot(
|
||
&mut associated_urls,
|
||
&r.twitter_screen_name,
|
||
q.id,
|
||
&m,
|
||
&t,
|
||
&r.toot_id,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if let Some(a) = &scootaloo_alt_services {
|
||
replace_alt_services(&mut associated_urls, a);
|
||
}
|
||
|
||
decode_urls(&mut status_text, &associated_urls);
|
||
}
|
||
|
||
// building associative media list
|
||
let (media_url, status_medias) =
|
||
generate_media_ids(tweet, &scootaloo_cache_path, &mastodon).await;
|
||
|
||
status_text = status_text.replace(&media_url, "");
|
||
|
||
// now that the text won’t be altered anymore, we can safely remove HTML
|
||
// entities
|
||
status_text = decode_html_entities(&status_text).to_string();
|
||
|
||
info!("Building corresponding Mastodon status");
|
||
|
||
let mut post_status = PostStatusInputOptions {
|
||
media_ids: None,
|
||
poll: None,
|
||
in_reply_to_id: None,
|
||
sensitive: None,
|
||
spoiler_text: None,
|
||
visibility: None,
|
||
scheduled_at: None,
|
||
language: None,
|
||
quote_id: None,
|
||
};
|
||
|
||
if !status_medias.is_empty() {
|
||
post_status.media_ids = Some(status_medias);
|
||
}
|
||
|
||
// thread if necessary
|
||
if tweet.in_reply_to_user_id.is_some() {
|
||
let lconn = task_conn.lock().await;
|
||
if let Ok(Some(r)) = read_state(
|
||
&lconn,
|
||
&mastodon_config.twitter_screen_name,
|
||
tweet.in_reply_to_status_id,
|
||
) {
|
||
post_status.in_reply_to_id = Some(r.toot_id.to_owned());
|
||
}
|
||
drop(lconn);
|
||
}
|
||
|
||
// language if any
|
||
if let Some(l) = &tweet.lang {
|
||
post_status.language = Some(l.to_string());
|
||
}
|
||
|
||
// can be activated for test purposes
|
||
// post_status.visibility = Some(megalodon::entities::StatusVisibility::Direct);
|
||
|
||
let published_status = mastodon
|
||
.post_status(status_text, Some(&post_status))
|
||
.await?
|
||
.json();
|
||
// this will return if it cannot publish the status preventing the last_tweet from
|
||
// being written into db
|
||
|
||
let ttt_towrite = TweetToToot {
|
||
twitter_screen_name: mastodon_config.twitter_screen_name.clone(),
|
||
tweet_id: tweet.id,
|
||
toot_id: published_status.id,
|
||
};
|
||
|
||
// write the current state (tweet ID and toot ID) to avoid copying it another time
|
||
let lconn = task_conn.lock().await;
|
||
write_state(&lconn, ttt_towrite)?;
|
||
drop(lconn);
|
||
}
|
||
Ok::<(), ScootalooError>(())
|
||
})
|
||
})
|
||
.buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
|
||
|
||
// launch and wait for every handle
|
||
while let Some(result) = stream.next().await {
|
||
match result {
|
||
Ok(Err(e)) => eprintln!("Error within thread: {e}"),
|
||
Err(e) => eprintln!("Error with thread: {e}"),
|
||
_ => (),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Copies the Twitter profile into Mastodon
|
||
#[tokio::main]
|
||
pub async fn profile(config: Config, bot: Option<bool>) {
|
||
let mut stream = futures::stream::iter(config.mastodon.into_values())
|
||
.map(|mastodon_config| {
|
||
let token = get_oauth2_token(&config.twitter);
|
||
|
||
spawn(async move {
|
||
// get the user of the last tweet of the feed
|
||
let twitter_user =
|
||
get_user_timeline(&mastodon_config.twitter_screen_name, &token, None, 1)
|
||
.await?
|
||
.first()
|
||
.ok_or_else(|| ScootalooError::new("Can’t extract a tweet from the feed!"))?
|
||
.clone()
|
||
.user
|
||
.ok_or_else(|| ScootalooError::new("No user in Tweet!"))?;
|
||
|
||
let note = get_note_from_description(
|
||
&twitter_user.description,
|
||
&twitter_user.entities.description.urls,
|
||
);
|
||
|
||
let fields_attributes = get_attribute_from_url(&twitter_user.entities.url);
|
||
|
||
let display_name = Some(String::from_utf16_lossy(
|
||
&twitter_user
|
||
.name
|
||
.encode_utf16()
|
||
.take(30)
|
||
.collect::<Vec<u16>>(),
|
||
));
|
||
|
||
let header = match twitter_user.profile_banner_url {
|
||
Some(h) => Some(base64_media(&h).await?),
|
||
None => None,
|
||
};
|
||
|
||
let update_creds = UpdateCredentialsInputOptions {
|
||
bot,
|
||
display_name,
|
||
note,
|
||
avatar: Some(
|
||
base64_media(&twitter_user.profile_image_url_https.replace("_normal", ""))
|
||
.await?,
|
||
),
|
||
header,
|
||
fields_attributes,
|
||
..Default::default()
|
||
};
|
||
|
||
let mastodon = get_mastodon_token(&mastodon_config);
|
||
|
||
mastodon.update_credentials(Some(&update_creds)).await?;
|
||
|
||
Ok::<(), ScootalooError>(())
|
||
})
|
||
})
|
||
.buffer_unordered(config.scootaloo.rate_limit.unwrap_or(DEFAULT_RATE_LIMIT));
|
||
|
||
while let Some(result) = stream.next().await {
|
||
match result {
|
||
Ok(Err(e)) => eprintln!("Error within thread: {e}"),
|
||
Err(e) => eprintln!("Error with thread: {e}"),
|
||
_ => (),
|
||
}
|
||
}
|
||
}
|