Merge branch '4-use-original-source-file-when-possible' into 'master'

✨: add peertube source video file

Closes #4

See merge request veretcle/tootube!19

Cargo.toml

@@ -1,21 +1,23 @@
 [package]
 name = "tootube"
 authors = ["VC <veretcle+framagit@mateu.be>"]
-version = "0.4.1"
+version = "0.6.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+rpassword = "^7.3"
 reqwest = { version = "^0.11", features = ["json", "stream", "multipart"] }
 tokio = { version = "^1", features = ["full"] }
 clap = "^4"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "^1", features = ["derive"] }
 toml = "^0.5"
 log = "^0.4"
 env_logger = "^0.10"
-futures-core = "0.3.28"
-bytes = "1.5.0"
+indicatif = "^0.17"
+async-stream = "^0.3"
+futures-util = "^0.3"
 
 [profile.release]
 strip = true

README.md

@@ -13,6 +13,7 @@ So consider this a work in progress that will slowly get better with time.
 # What does it do exactly?
 
 * it retrieves the latest PeerTube video download URL from an instance
+* if you register the app into your PeerTube instance (optional), you can retrieve the latest video source instead of the highest quality encoded video
 * it creates a resumable upload into the target YouTube account
 * it downloads/uploads the latest PeerTube video to YouTube without using a cache (stream-to-stream)
 
@@ -46,6 +47,15 @@ Create your `tootube.toml` config file:
 ```toml
 [peertube]
 base_url="https://p.nintendojo.fr"
+# optional
+# allows you to delete the original video source file once it’s uploaded to YouTube
+delete_video_source_after_transfer=true # this option is only available if you have Administrator privileges
+# optional
+# everything below is given by the register command with --peertube option
+[peertube.oauth2]
+client_id="<YOUR CLIENT_ID>"
+client_secret="<YOUR CLIENT_SECRET"
+refresh_token="/var/lib/tootube/refresh_token" # refresh_token are single use only in PeerTube so we need to store it in a separate file
 
 [youtube]
 refresh_token="" # leave empty for now
@@ -56,7 +66,19 @@ client_secret="<YOUR CLIENT_SECRET>"
 Then run:
 
 ```bash
-tootube register --config <PATH TO YOUR TOOTUBE.TOML FILE>
+tootube register --youtube --config <PATH TO YOUR TOOTUBE.TOML FILE>
 ```
 
 You’ll be then prompted with all the necessary information to register `tootube`. You’ll end with a `refresh_token` that you can now paste inside your tootube configuration.
+
+If you wish to register `tootube` on PeerTube, you can do so by using:
+
+```bash
+tootube register --peertube --config <PATH TO YOUR TOOTUBE.TOML FILE>
+```
+
+It will require your username/password (beware that 2FA is not supported for this feature as of now) and generate a first `refresh_token` that you will put inside the aformentioned file:
+
+```bash
+echo -n '<REFRESH_TOKEN>' > /var/lib/tootube/refresh_token
+```

src/config.rs

@@ -6,11 +6,33 @@ use std::fs::read_to_string;
 pub struct Config {
     pub peertube: PeertubeConfig,
     pub youtube: YoutubeConfig,
+    #[serde(default)]
+    pub tootube: TootubeConfig,
 }
 
 #[derive(Debug, Deserialize)]
 pub struct PeertubeConfig {
     pub base_url: String,
+    #[serde(default)]
+    pub delete_video_source_after_transfer: bool,
+    pub oauth2: Option<PeertubeConfigOauth2>,
+}
+
+impl Default for PeertubeConfig {
+    fn default() -> Self {
+        PeertubeConfig {
+            base_url: "".to_string(),
+            delete_video_source_after_transfer: false,
+            oauth2: None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct PeertubeConfigOauth2 {
+    pub client_id: String,
+    pub client_secret: String,
+    pub refresh_token: String,
 }
 
 #[derive(Debug, Deserialize)]
@@ -20,6 +42,21 @@ pub struct YoutubeConfig {
     pub client_secret: String,
 }
 
+#[derive(Debug, Deserialize)]
+pub struct TootubeConfig {
+    pub progress_bar: String,
+    pub progress_chars: String,
+}
+
+impl Default for TootubeConfig {
+    fn default() -> Self {
+        TootubeConfig {
+            progress_bar: "{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})".to_string(),
+            progress_chars: "#>-".to_string(),
+        }
+    }
+}
+
 /// Parses the TOML file into a Config struct
 pub fn parse_toml(toml_file: &str) -> Config {
     let toml_config =

src/lib.rs

@@ -1,7 +1,4 @@
-use bytes::Bytes;
-use futures_core::stream::Stream;
 use log::debug;
-use std::error::Error;
 
 mod error;
 
@@ -10,18 +7,13 @@ pub use config::parse_toml;
 use config::Config;
 
 mod peertube;
-use peertube::{get_latest_video, get_max_resolution_dl};
+pub use peertube::register as register_peertube;
+use peertube::{delete_original_video_source, get_latest_video, get_original_video_source};
 
 mod youtube;
-pub use youtube::register;
+pub use youtube::register as register_youtube;
 use youtube::{add_video_to_playlists, create_resumable_upload, now_kiss};
 
-async fn get_dl_video_stream(
-    u: &str,
-) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>, Box<dyn Error>> {
-    Ok(reqwest::get(u).await?.bytes_stream())
-}
-
 #[tokio::main]
 pub async fn run(config: Config, pl: Vec<String>) {
     // Get the latest video object
@@ -31,21 +23,44 @@ pub async fn run(config: Config, pl: Vec<String>) {
             panic!("Cannot retrieve the latest video, something must have gone terribly wrong: {e}")
         });
 
-    let dl_url = get_max_resolution_dl(latest_vid.streaming_playlists.as_ref().unwrap());
-    debug!("PT download URL: {}", &dl_url);
+    // We have a refresh_token, try to use it
+    let source_url = match &config.peertube.oauth2 {
+        Some(_) => get_original_video_source(&latest_vid.uuid, &config.peertube)
+            .await
+            .ok(),
+        None => None,
+    };
 
-    let pt_stream = get_dl_video_stream(&dl_url)
-        .await
-        .unwrap_or_else(|e| panic!("Cannot download video at URL {}: {}", dl_url, e));
+    // Whatever happens, collect the highest quality possible
+    let high_quality_url = latest_vid.streaming_playlists.as_ref().unwrap()[0]
+        .files
+        .iter()
+        .max()
+        .unwrap()
+        .file_download_url
+        .clone();
+
+    // dl_url corresponds to source url if available, best quality if not
+    let dl_url = match source_url {
+        Some(s) => s,
+        None => high_quality_url,
+    };
+
+    debug!("PT download URL: {}", &dl_url);
 
     let resumable_upload_url = create_resumable_upload(&config.youtube, &latest_vid)
         .await
         .unwrap_or_else(|e| panic!("Cannot retrieve the upload’s resumable id: {e}"));
     debug!("YT upload URL: {}", &resumable_upload_url);
 
-    let yt_video_id = now_kiss(pt_stream, &resumable_upload_url, &config.youtube)
-        .await
-        .unwrap_or_else(|e| panic!("Cannot resume upload!: {e}"));
+    let yt_video_id = now_kiss(
+        &dl_url,
+        &resumable_upload_url,
+        &config.youtube,
+        &config.tootube,
+    )
+    .await
+    .unwrap_or_else(|e| panic!("Cannot resume upload!: {e}"));
     debug!("YT video ID: {}", &yt_video_id);
 
     if !pl.is_empty() {
@@ -53,4 +68,13 @@ pub async fn run(config: Config, pl: Vec<String>) {
             .await
             .unwrap_or_else(|e| panic!("Cannot add video to playlist(s): {e}"));
     }
+
+    // delete the source video if requested (it won’t be used anymore)
+    if config.peertube.delete_video_source_after_transfer {
+        delete_original_video_source(&latest_vid.uuid, &config.peertube)
+            .await
+            .unwrap_or_else(|e| panic!("Cannot delete source video: {e}"));
+
+        debug!("Original Video {} has been deleted", &latest_vid.uuid);
+    }
 }

src/main.rs

@@ -10,6 +10,7 @@ fn main() {
         .arg(
             Arg::new("config")
                 .short('c')
+                .global(true)
                 .long("config")
                 .value_name("CONFIG_FILE")
                 .help("TOML config file for tootube")
@@ -26,31 +27,57 @@ fn main() {
                 .num_args(0..)
                 .display_order(2),
         )
+        .arg(
+            Arg::new("vice")
+                .long("vice")
+                .aliases(["chybrare", "coquinou"])
+                .action(clap::ArgAction::SetTrue)
+                .display_order(3),
+        )
         .subcommand(
             Command::new("register")
                 .version(env!("CARGO_PKG_VERSION"))
-                .about("Command to register to YouTube OAuth2.0")
+                .about("Command to register to YouTube or PeerTube OAuth2.0")
                 .arg(
-                    Arg::new("config")
-                        .short('c')
-                        .long("config")
-                        .value_name("CONFIG_FILE")
-                        .help("TOML config file for tootube")
-                        .num_args(1)
-                        .default_value(DEFAULT_CONFIG_PATH)
-                        .display_order(1),
+                    Arg::new("youtube")
+                        .long("youtube")
+                        .short('y')
+                        .required(true)
+                        .conflicts_with("peertube")
+                        .action(clap::ArgAction::SetTrue),
+                )
+                .arg(
+                    Arg::new("peertube")
+                        .long("peertube")
+                        .short('p')
+                        .required(true)
+                        .action(clap::ArgAction::SetTrue),
                 ),
         )
         .get_matches();
 
     if let Some(("register", sub_m)) = matches.subcommand() {
         let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
-        register(&config.youtube)
-            .unwrap_or_else(|e| panic!("Cannot register to YouTube API: {}", e));
+
+        if sub_m.get_flag("youtube") {
+            register_youtube(&config.youtube)
+                .unwrap_or_else(|e| panic!("Cannot register to YouTube API: {}", e));
+        }
+
+        if sub_m.get_flag("peertube") {
+            register_peertube(&config.peertube)
+                .unwrap_or_else(|e| panic!("Cannot register to PeerTube API: {}", e));
+        }
+
         return;
     }
 
-    let config = parse_toml(matches.get_one::<String>("config").unwrap());
+    let mut config = parse_toml(matches.get_one::<String>("config").unwrap());
+
+    if matches.get_flag("vice") {
+        config.tootube.progress_bar = "{msg}\n[{elapsed_precise}] 8{wide_bar:.magenta}ᗺ {bytes}/{total_bytes} ({bytes_per_sec}, {eta})".to_string();
+        config.tootube.progress_chars = "=D ".to_string();
+    }
 
     let playlists: Vec<String> = matches
         .get_many::<String>("playlists")

src/peertube.rs

@@ -1,5 +1,10 @@
-use serde::Deserialize;
-use std::{boxed::Box, cmp::Ordering, error::Error};
+use crate::{config::PeertubeConfig, error::TootubeError};
+use log::debug;
+use reqwest::Client;
+use rpassword::prompt_password;
+use serde::{Deserialize, Serialize};
+use std::{boxed::Box, cmp::Ordering, error::Error, io::stdin};
+use tokio::fs::{read_to_string, write};
 
 #[derive(Debug, Deserialize)]
 pub struct PeerTubeVideos {
@@ -22,6 +27,44 @@ pub struct PeerTubeVideoStreamingPlaylists {
     pub files: Vec<PeerTubeVideoStreamingPlaylistsFiles>,
 }
 
+#[derive(Debug, Deserialize)]
+struct PeerTubeOauthClientsLocalResponse {
+    client_id: String,
+    client_secret: String,
+}
+
+#[derive(Debug, Serialize)]
+struct PeerTubeUsersToken {
+    client_id: String,
+    client_secret: String,
+    grant_type: String,
+    password: Option<String>,
+    username: Option<String>,
+    refresh_token: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct PeerTubeUsersTokenResponse {
+    refresh_token: String,
+    access_token: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct PeerTubeVideoSourceResponse {
+    #[serde(rename = "fileDownloadUrl")]
+    pub file_download_url: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct PeerTubeVideoTokenResponse {
+    files: PeerTubeVideoTokenResponseFiles,
+}
+
+#[derive(Debug, Deserialize)]
+struct PeerTubeVideoTokenResponseFiles {
+    token: String,
+}
+
 #[derive(Eq, Debug, Deserialize)]
 pub struct PeerTubeVideoStreamingPlaylistsFiles {
     pub id: u64,
@@ -65,11 +108,6 @@ pub async fn get_latest_video(u: &str) -> Result<PeerTubeVideo, Box<dyn Error>>
     Ok(vid)
 }
 
-/// This returns the direct download URL for with the maximum resolution
-pub fn get_max_resolution_dl(p: &[PeerTubeVideoStreamingPlaylists]) -> String {
-    p[0].files.iter().max().unwrap().file_download_url.clone()
-}
-
 /// This gets all the crispy details about one particular video
 async fn get_video_detail(u: &str, v: &str) -> Result<PeerTubeVideo, Box<dyn Error>> {
     let body = reqwest::get(format!("{}/api/v1/videos/{}", u, v))
@@ -80,33 +118,162 @@ async fn get_video_detail(u: &str, v: &str) -> Result<PeerTubeVideo, Box<dyn Err
     Ok(body)
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
+/// This function makes the registration process a little bit easier
+#[tokio::main]
+pub async fn register(config: &PeertubeConfig) -> Result<(), Box<dyn Error>> {
+    // Get client ID/secret
+    let oauth2_client = reqwest::get(format!("{}/api/v1/oauth-clients/local", config.base_url))
+        .await?
+        .json::<PeerTubeOauthClientsLocalResponse>()
+        .await?;
 
-    #[test]
-    fn test_get_max_resolution_dl() {
-        let str_plist: Vec<PeerTubeVideoStreamingPlaylists> =
-            vec![PeerTubeVideoStreamingPlaylists {
-                files: vec![
-                    PeerTubeVideoStreamingPlaylistsFiles {
-                        id: 1025,
-                        resolution: PeerTubeVideoStreamingPlaylistsFilesResolution { id: 990 },
-                        file_download_url: "meh!".to_string(),
-                    },
-                    PeerTubeVideoStreamingPlaylistsFiles {
-                        id: 1027,
-                        resolution: PeerTubeVideoStreamingPlaylistsFilesResolution { id: 1720 },
-                        file_download_url: "blah".to_string(),
-                    },
-                    PeerTubeVideoStreamingPlaylistsFiles {
-                        id: 1026,
-                        resolution: PeerTubeVideoStreamingPlaylistsFilesResolution { id: 360 },
-                        file_download_url: "nope".to_string(),
-                    },
-                ],
-            }];
+    println!(
+        "Please type your PeerTube username for instance {}:",
+        config.base_url
+    );
+    let mut username = String::new();
+    stdin()
+        .read_line(&mut username)
+        .expect("Unable to read back the username!");
 
-        assert_eq!("blah".to_string(), get_max_resolution_dl(&str_plist));
-    }
+    let password = prompt_password("Your password: ").expect("Unable to read back the password!");
+
+    let params = PeerTubeUsersToken {
+        client_id: oauth2_client.client_id.clone(),
+        client_secret: oauth2_client.client_secret.clone(),
+        grant_type: "password".to_string(),
+        username: Some(username.trim().to_string()),
+        password: Some(password.clone()),
+        refresh_token: None,
+    };
+
+    let client = Client::new();
+    let oauth2_token = client
+        .post(format!("{}/api/v1/users/token", config.base_url))
+        .form(&params)
+        .send()
+        .await?
+        .json::<PeerTubeUsersTokenResponse>()
+        .await?;
+
+    println!("You can now paste the following lines inside the `peertube` section of your tootube.toml file:");
+
+    println!();
+
+    println!("[peertube.oauth2]");
+    println!("client_id=\"{}\"", oauth2_client.client_id);
+    println!("client_secret=\"{}\"", oauth2_client.client_secret);
+    println!("refresh_token=<path to refresh_token>");
+
+    println!();
+
+    println!("Finally, add the refresh token inside the refresh_token path:");
+    println!("{}", oauth2_token.refresh_token);
+
+    Ok(())
+}
+
+async fn get_refresh_token(config: &PeertubeConfig) -> Result<String, Box<dyn Error>> {
+    // unwrap is safe here, this function is only called when oauth2 is present
+    let oauth2_config = config.oauth2.as_ref().unwrap().clone();
+    // retrieve the refresh_token from the file
+    let refresh_token = read_to_string(&oauth2_config.refresh_token).await?;
+
+    debug!(
+        "Opened file {} to retrieve Token",
+        &oauth2_config.refresh_token
+    );
+
+    let params = PeerTubeUsersToken {
+        client_id: oauth2_config.client_id.clone(),
+        client_secret: oauth2_config.client_secret.clone(),
+        grant_type: "refresh_token".to_string(),
+        refresh_token: Some(refresh_token),
+        username: None,
+        password: None,
+    };
+
+    let client = Client::new();
+    let oauth2_token = client
+        .post(format!("{}/api/v1/users/token", config.base_url))
+        .form(&params)
+        .send()
+        .await?
+        .json::<PeerTubeUsersTokenResponse>()
+        .await?;
+
+    debug!("Retrieved access_token: {}", &oauth2_token.access_token);
+
+    // write the new refresh token to the file
+    write(&oauth2_config.refresh_token, oauth2_token.refresh_token).await?;
+
+    debug!(
+        "Written refresh_token to file {}",
+        &oauth2_config.refresh_token
+    );
+
+    Ok(oauth2_token.access_token)
+}
+
+pub async fn get_original_video_source(
+    uuid: &str,
+    config: &PeertubeConfig,
+) -> Result<String, Box<dyn Error>> {
+    let access_token = get_refresh_token(config).await?;
+
+    let client = Client::new();
+
+    let source_vid = client
+        .get(format!("{}/api/v1/videos/{}/source", config.base_url, uuid))
+        .header("Authorization", format!("Bearer {}", &access_token))
+        .send()
+        .await?
+        .json::<PeerTubeVideoSourceResponse>()
+        .await?;
+
+    debug!("Got the Source Vid URL: {}", &source_vid.file_download_url);
+
+    let video_file_token = client
+        .post(format!("{}/api/v1/videos/{}/token", config.base_url, uuid))
+        .header("Authorization", format!("Bearer {}", &access_token))
+        .send()
+        .await?
+        .json::<PeerTubeVideoTokenResponse>()
+        .await?;
+
+    debug!("Got the File Token: {}", &video_file_token.files.token);
+
+    Ok(format!(
+        "{}?videoFileToken={}",
+        source_vid.file_download_url, video_file_token.files.token
+    ))
+}
+
+pub async fn delete_original_video_source(
+    uuid: &str,
+    config: &PeertubeConfig,
+) -> Result<(), Box<dyn Error>> {
+    let access_token = get_refresh_token(config).await?;
+
+    let client = Client::new();
+
+    let res = client
+        .delete(format!(
+            "{}/api/v1/videos/{}/source/file",
+            config.base_url, uuid
+        ))
+        .header("Authorization", format!("Bearer {}", &access_token))
+        .send()
+        .await?;
+
+    if !res.status().is_success() {
+        return Err(TootubeError::new(&format!(
+            "Cannot delete source video file {}: {}",
+            uuid,
+            res.text().await?
+        ))
+        .into());
+    }
+
+    Ok(())
 }

src/youtube.rs

@@ -1,10 +1,15 @@
-use crate::{config::YoutubeConfig, error::TootubeError, peertube::PeerTubeVideo};
-use bytes::Bytes;
-use futures_core::stream::Stream;
-use log::debug;
+use crate::{
+    config::{TootubeConfig, YoutubeConfig},
+    error::TootubeError,
+    peertube::PeerTubeVideo,
+};
+use async_stream::stream;
+use futures_util::StreamExt;
+use indicatif::{ProgressBar, ProgressStyle};
+use log::{debug, warn};
 use reqwest::{multipart::Form, Body, Client};
 use serde::{Deserialize, Serialize};
-use std::{error::Error, io::stdin};
+use std::{cmp::min, error::Error, io::stdin};
 use tokio::sync::OnceCell;
 
 static ACCESS_TOKEN: OnceCell<String> = OnceCell::const_new();
@@ -248,7 +253,7 @@ async fn get_playlist_ids(
         // if nextPageToken is present, continue the loop
         match local_pl.next_page_token {
             None => break,
-            Some(a) => page_token = a.clone(),
+            Some(a) => page_token.clone_from(&a),
         }
     }
 
@@ -304,10 +309,17 @@ pub async fn create_resumable_upload(
 ) -> Result<String, Box<dyn Error>> {
     let access_token = refresh_token(config).await?;
 
+    if vid.name.chars().count() > 100 {
+        warn!(
+            "PT Video Title ({}) is too long, it will be truncated",
+            &vid.name
+        );
+    }
+
     let upload_params = YoutubeUploadParams {
         snippet: {
             YoutubeUploadParamsSnippet {
-                title: vid.name.clone(),
+                title: vid.name.chars().take(100).collect::<String>(),
                 description: vid.description.clone(),
                 tags: vid.tags.clone(),
                 ..Default::default()
@@ -340,23 +352,53 @@ pub async fn create_resumable_upload(
 }
 
 /// This takes the PT stream for download, connects it to YT stream for upload
-pub async fn now_kiss<'a>(
-    stream: impl Stream<Item = Result<Bytes, reqwest::Error>>
-        + std::marker::Send
-        + std::marker::Sync
-        + 'a + 'static,
-    r_url: &'a str,
-    config: &'a YoutubeConfig,
+pub async fn now_kiss(
+    dl_url: &str,
+    r_url: &str,
+    config: &YoutubeConfig,
+    pg_conf: &TootubeConfig,
 ) -> Result<String, Box<dyn Error>> {
     // Get access token
     let access_token = refresh_token(config).await?;
 
+    // Get the upstream bytes stream
+    let res = reqwest::get(dl_url).await?;
+    let content_lengh = res
+        .content_length()
+        .ok_or(format!("Cannot get content length from {}", dl_url))?;
+    let mut stream = res.bytes_stream();
+
+    // Create the progress bar
+    let pb = ProgressBar::new(content_lengh);
+    pb.set_style(
+        ProgressStyle::default_bar()
+            .template(&pg_conf.progress_bar)?
+            .progress_chars(&pg_conf.progress_chars),
+    );
+    pb.set_message("Transferring…");
+    let mut transferring: u64 = 0;
+
+    // yields the stream chunk by chunk, updating the progress bar at the same time
+    let async_stream = stream! {
+        while let Some(chunk) = stream.next().await {
+            if let Ok(chunk) = &chunk {
+                let new = min(transferring + (chunk.len() as u64), content_lengh);
+                transferring = new;
+                pb.set_position(new);
+                if transferring >= content_lengh {
+                    pb.finish();
+                }
+            }
+            yield chunk;
+        }
+    };
+
     // Create client
     let client = Client::new();
     let res = client
         .put(r_url)
         .header("Authorization", format!("Bearer {}", access_token))
-        .body(Body::wrap_stream(stream))
+        .body(Body::wrap_stream(async_stream))
         .send()
         .await?;