Merge branch '2-tootube-should-log-moar' into 'master'

Resolve "Tootube should log moar"

Closes #2

See merge request veretcle/tootube!12

Cargo.lock

@@ -530,6 +530,16 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "mime_guess"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -760,6 +770,7 @@ dependencies = [
  "js-sys",
  "log",
  "mime",
+ "mime_guess",
  "native-tls",
  "once_cell",
  "percent-encoding",
@@ -1070,7 +1081,7 @@ dependencies = [
 
 [[package]]
 name = "tootube"
-version = "0.3.0"
+version = "0.4.1"
 dependencies = [
  "bytes",
  "clap",
@@ -1115,6 +1126,15 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
+[[package]]
+name = "unicase"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.13"
@@ -1159,6 +1179,12 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
 [[package]]
 name = "want"
 version = "0.3.1"

Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "tootube"
 authors = ["VC <veretcle+framagit@mateu.be>"]
-version = "0.3.0"
+version = "0.4.1"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { version = "^0.11", features = ["json", "stream"] }
+reqwest = { version = "^0.11", features = ["json", "stream", "multipart"] }
 tokio = { version = "^1", features = ["full"] }
 clap = "^4"
 serde = { version = "1.0", features = ["derive"] }

README.md

@@ -6,50 +6,57 @@ This program takes the very last video published on a PeerTube instance and push
 
 This is an early prototype not really suited for production purposes for now:
 * it still relies way too much on pre-determined value to upload the video to YouTube
-* it cannot determine the playlists it needs to put them in
 * it cannot determine the recording date (believe me, I tried!)
-* there are still a lot of static values that I would rather not have static (like the cache directory…)
-* it is 100% sync, meaning it’s clearly not optimal for now and probably won’t be for the next releases
 
-So consider this a work in progress that will slowly get better with time
+So consider this a work in progress that will slowly get better with time.
 
 # What does it do exactly?
 
-* it downloads the latest PeerTube video from an instance
+* it retrieves the latest PeerTube video download URL from an instance
-* stores it in cache directory
+* it creates a resumable upload into the target YouTube account
-* uploads it to YouTube
+* it downloads/uploads the latest PeerTube video to YouTube without using a cache (stream-to-stream)
 
 # What doesn’t it do exactly?
 
-* it cannot register the original key (see below)
-* it relies on a local cache despite the fact that it might well be possible to just download from PT/upload to YT at the same time (I don’t see why not in fact)
 * it doesn’t retrieve ALL the original PeerTube video properties like licences, languages, categories, etc… again: early prototype
 
-# Obtain Authorization Token from Google
+# Howtos
+## General usage
 
-That the complicated part:
+Once you fill your `tootube.toml` config file (see below), you can run `tootube` like this:
-* create an OAuth2.0 application with authorization for Youtube DATA Api v3 Upload
+
+```bash
+tootube --config tootube.toml --playlist playlist_1 "Things You Might Like"
+```
+
+You can turn on debug using env var `RUST_LOG`.
+
+## Obtain Authorization Token from Google
+
+The complicated part:
+* create an OAuth2.0 application with authorization for Youtube DATA Api v3 Upload and Youtube DATA Api v3 (generally referenced as `../auth/youtube.upload` and `../auth/youtube`)
 * create a OAuth2.0 client with Desktop client
 
 You’ll need:
 * the `client_id` from your OAuth2.0 client
 * the `client_secret` from you OAuth2.0 client
 
-Then enter in:
+Create your `tootube.toml` config file:
 
-```
+```toml
-https://accounts.google.com/o/oauth2/v2/auth?client_id=XXX.apps.googleusercontent.com&redirect_uri=urn:ietf:wg:oauth:2.0:oob&scope=https://www.googleapis.com/auth/youtube.upload&response_type=code
+[peertube]
+base_url="https://p.nintendojo.fr"
+
+[youtube]
+refresh_token="" # leave empty for now
+client_id="<YOUR CLIENT_ID>"
+client_secret="<YOUR CLIENT_SECRET>"
 ```
 
-And accept that your YouTube account might be modified. You’ll get a code then; enter this `curl` post:
+Then run:
 
-```
+```bash
-curl -s \
+tootube register --config <PATH TO YOUR TOOTUBE.TOML FILE>
---request POST \
---data "code=[THE_CODE]&client_id=XXX.apps.googleusercontent.com&client_secret=[THE_CLIENT_SECRET]&redirect_uri=urn:ietf:wg:oauth:2.0:oob&grant_type=authorization_code" \
-https://accounts.google.com/o/oauth2/token
 ```
 
-You’ll get a Token. The only important part is the `refresh_token`
+You’ll be then prompted with all the necessary information to register `tootube`. You’ll end with a `refresh_token` that you can now paste inside your tootube configuration.
-
-In your `tootube.toml` config file, put the `refresh_token`.

src/lib.rs

@@ -1,5 +1,6 @@
 use bytes::Bytes;
 use futures_core::stream::Stream;
+use log::debug;
 use std::error::Error;
 
 mod error;
@@ -12,7 +13,8 @@ mod peertube;
 use peertube::{get_latest_video, get_max_resolution_dl};
 
 mod youtube;
-use youtube::{create_resumable_upload, now_kiss};
+pub use youtube::register;
+use youtube::{add_video_to_playlists, create_resumable_upload, now_kiss};
 
 async fn get_dl_video_stream(
     u: &str,
@@ -21,7 +23,7 @@ async fn get_dl_video_stream(
 }
 
 #[tokio::main]
-pub async fn run(config: Config) {
+pub async fn run(config: Config, pl: Vec<String>) {
     // Get the latest video object
     let latest_vid = get_latest_video(&config.peertube.base_url)
         .await
@@ -30,6 +32,7 @@ pub async fn run(config: Config) {
         });
 
     let dl_url = get_max_resolution_dl(latest_vid.streaming_playlists.as_ref().unwrap());
+    debug!("PT download URL: {}", &dl_url);
 
     let pt_stream = get_dl_video_stream(&dl_url)
         .await
@@ -38,8 +41,16 @@ pub async fn run(config: Config) {
     let resumable_upload_url = create_resumable_upload(&config.youtube, &latest_vid)
         .await
         .unwrap_or_else(|e| panic!("Cannot retrieve the upload’s resumable id: {e}"));
+    debug!("YT upload URL: {}", &resumable_upload_url);
 
-    now_kiss(pt_stream, &resumable_upload_url, &config.youtube)
+    let yt_video_id = now_kiss(pt_stream, &resumable_upload_url, &config.youtube)
         .await
         .unwrap_or_else(|e| panic!("Cannot resume upload!: {e}"));
+    debug!("YT video ID: {}", &yt_video_id);
+
+    if !pl.is_empty() {
+        add_video_to_playlists(&config.youtube, &yt_video_id, &pl)
+            .await
+            .unwrap_or_else(|e| panic!("Cannot add video to playlist(s): {e}"));
+    }
 }

src/main.rs

@@ -17,11 +17,48 @@ fn main() {
                 .default_value(DEFAULT_CONFIG_PATH)
                 .display_order(1),
         )
+        .arg(
+            Arg::new("playlists")
+                .short('p')
+                .long("playlist")
+                .value_name("PLAYLIST")
+                .help("List of playlists to add the video to")
+                .num_args(0..)
+                .display_order(2),
+        )
+        .subcommand(
+            Command::new("register")
+                .version(env!("CARGO_PKG_VERSION"))
+                .about("Command to register to YouTube OAuth2.0")
+                .arg(
+                    Arg::new("config")
+                        .short('c')
+                        .long("config")
+                        .value_name("CONFIG_FILE")
+                        .help("TOML config file for tootube")
+                        .num_args(1)
+                        .default_value(DEFAULT_CONFIG_PATH)
+                        .display_order(1),
+                ),
+        )
         .get_matches();
 
+    if let Some(("register", sub_m)) = matches.subcommand() {
+        let config = parse_toml(sub_m.get_one::<String>("config").unwrap());
+        register(&config.youtube)
+            .unwrap_or_else(|e| panic!("Cannot register to YouTube API: {}", e));
+        return;
+    }
+
     let config = parse_toml(matches.get_one::<String>("config").unwrap());
 
+    let playlists: Vec<String> = matches
+        .get_many::<String>("playlists")
+        .unwrap_or_default()
+        .map(|v| v.to_string())
+        .collect();
+
     env_logger::init();
 
-    run(config);
+    run(config, playlists);
 }

src/peertube.rs

@@ -1,5 +1,5 @@
 use serde::Deserialize;
-use std::{boxed::Box, error::Error};
+use std::{boxed::Box, cmp::Ordering, error::Error};
 
 #[derive(Debug, Deserialize)]
 pub struct PeerTubeVideos {
@@ -22,7 +22,7 @@ pub struct PeerTubeVideoStreamingPlaylists {
     pub files: Vec<PeerTubeVideoStreamingPlaylistsFiles>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Eq, Debug, Deserialize)]
 pub struct PeerTubeVideoStreamingPlaylistsFiles {
     pub id: u64,
     pub resolution: PeerTubeVideoStreamingPlaylistsFilesResolution,
@@ -30,7 +30,25 @@ pub struct PeerTubeVideoStreamingPlaylistsFiles {
     pub file_download_url: String,
 }
 
-#[derive(Debug, Deserialize)]
+impl Ord for PeerTubeVideoStreamingPlaylistsFiles {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.resolution.id.cmp(&other.resolution.id)
+    }
+}
+
+impl PartialOrd for PeerTubeVideoStreamingPlaylistsFiles {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for PeerTubeVideoStreamingPlaylistsFiles {
+    fn eq(&self, other: &Self) -> bool {
+        self.resolution.id == other.resolution.id
+    }
+}
+
+#[derive(Eq, Debug, Deserialize, PartialEq)]
 pub struct PeerTubeVideoStreamingPlaylistsFilesResolution {
     pub id: u16,
 }
@@ -49,17 +67,7 @@ pub async fn get_latest_video(u: &str) -> Result<PeerTubeVideo, Box<dyn Error>>
 
 /// This returns the direct download URL for with the maximum resolution
 pub fn get_max_resolution_dl(p: &[PeerTubeVideoStreamingPlaylists]) -> String {
-    let mut res = 0;
+    p[0].files.iter().max().unwrap().file_download_url.clone()
-    let mut dl_url = String::new();
-
-    for i in p[0].files.iter() {
-        if i.resolution.id > res {
-            res = i.resolution.id;
-            dl_url = i.file_download_url.clone();
-        }
-    }
-
-    dl_url
 }
 
 /// This gets all the crispy details about one particular video

src/youtube.rs

@@ -1,9 +1,10 @@
 use crate::{config::YoutubeConfig, error::TootubeError, peertube::PeerTubeVideo};
 use bytes::Bytes;
 use futures_core::stream::Stream;
-use reqwest::{Body, Client};
+use log::debug;
+use reqwest::{multipart::Form, Body, Client};
 use serde::{Deserialize, Serialize};
-use std::error::Error;
+use std::{error::Error, io::stdin};
 use tokio::sync::OnceCell;
 
 static ACCESS_TOKEN: OnceCell<String> = OnceCell::const_new();
@@ -34,6 +35,11 @@ struct AccessTokenResponse {
     access_token: String,
 }
 
+#[derive(Deserialize, Debug)]
+struct RegistrationAccessTokenResponse {
+    refresh_token: String,
+}
+
 #[derive(Serialize, Debug)]
 struct YoutubeUploadParams {
     snippet: YoutubeUploadParamsSnippet,
@@ -82,6 +88,106 @@ impl Default for YoutubeUploadParamsStatus {
     }
 }
 
+#[derive(Serialize, Debug)]
+struct YoutubePlaylistItemsParams {
+    snippet: YoutubePlaylistItemsParamsSnippet,
+}
+
+#[derive(Serialize, Debug)]
+struct YoutubePlaylistItemsParamsSnippet {
+    #[serde(rename = "playlistId")]
+    playlist_id: String,
+    position: u16,
+    #[serde(rename = "resourceId")]
+    resource_id: YoutubePlaylistItemsParamsSnippetResourceId,
+}
+
+impl Default for YoutubePlaylistItemsParamsSnippet {
+    fn default() -> Self {
+        YoutubePlaylistItemsParamsSnippet {
+            playlist_id: "".to_string(),
+            position: 0,
+            resource_id: YoutubePlaylistItemsParamsSnippetResourceId {
+                ..Default::default()
+            },
+        }
+    }
+}
+
+#[derive(Serialize, Debug)]
+struct YoutubePlaylistItemsParamsSnippetResourceId {
+    kind: String,
+    #[serde(rename = "videoId")]
+    video_id: String,
+}
+
+impl Default for YoutubePlaylistItemsParamsSnippetResourceId {
+    fn default() -> Self {
+        YoutubePlaylistItemsParamsSnippetResourceId {
+            kind: "youtube#video".to_string(),
+            video_id: "".to_string(),
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+struct YoutubeVideos {
+    id: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct YoutubePlaylistListResponse {
+    #[serde(rename = "nextPageToken")]
+    next_page_token: Option<String>,
+    items: Vec<YoutubePlaylistListResponseItem>,
+}
+
+#[derive(Deserialize, Debug)]
+struct YoutubePlaylistListResponseItem {
+    id: String,
+    snippet: YoutubePlaylistListResponseItemSnippet,
+}
+
+#[derive(Deserialize, Debug)]
+struct YoutubePlaylistListResponseItemSnippet {
+    title: String,
+}
+
+/// This function makes the registration process a little bit easier
+#[tokio::main]
+pub async fn register(config: &YoutubeConfig) -> Result<(), Box<dyn Error>> {
+    println!("Click on the link below to authorize {} to upload to YouTube and deal with your playlists:", env!("CARGO_PKG_NAME"));
+    println!("https://accounts.google.com/o/oauth2/v2/auth?client_id={}&redirect_uri=urn:ietf:wg:oauth:2.0:oob&scope=https://www.googleapis.com/auth/youtube%20https://www.googleapis.com/auth/youtube.upload&response_type=code", config.client_id);
+    println!("Paste the returned authorization code:");
+
+    let mut input = String::new();
+    stdin()
+        .read_line(&mut input)
+        .expect("Unable to read back the authorization code!");
+
+    let form = Form::new()
+        .text("code", input)
+        .text("client_id", config.client_id.clone())
+        .text("client_secret", config.client_secret.clone())
+        .text("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
+        .text("grant_type", "authorization_code");
+
+    let client = Client::new();
+    let res = client
+        .post("https://accounts.google.com/o/oauth2/token")
+        .multipart(form)
+        .send()
+        .await?;
+
+    let refresh_token: RegistrationAccessTokenResponse = res.json().await?;
+
+    println!("You can now paste the following line inside the `youtube` section of your tootube.toml file:");
+
+    println!("refresh_token=\"{}\"", refresh_token.refresh_token);
+
+    Ok(())
+}
+
 /// Ensures that Token has been refreshed and that it is unique
 async fn refresh_token(config: &YoutubeConfig) -> Result<String, reqwest::Error> {
     ACCESS_TOKEN
@@ -102,12 +208,96 @@ async fn refresh_token(config: &YoutubeConfig) -> Result<String, reqwest::Error>
 
             let access_token: AccessTokenResponse = res.json().await?;
 
+            debug!("YT Access Token: {}", &access_token.access_token);
             Ok(access_token.access_token)
         })
         .await
         .cloned()
 }
 
+/// This function takes a list of playlists keyword and returns a list of playlist ID
+async fn get_playlist_ids(
+    config: &YoutubeConfig,
+    pl: &[String],
+) -> Result<Vec<String>, Box<dyn Error>> {
+    let mut page_token = String::new();
+    let mut playlists: Vec<String> = vec![];
+
+    let access_token = refresh_token(config).await?;
+    let client = Client::new();
+
+    while let Ok(local_pl) = client
+        .get(&format!(
+            "https://www.googleapis.com/youtube/v3/playlists?part=snippet&mine=true&pageToken={}",
+            page_token
+        ))
+        .header("Authorization", format!("Bearer {}", access_token))
+        .send()
+        .await?
+        .json::<YoutubePlaylistListResponse>()
+        .await
+    {
+        playlists.append(
+            &mut local_pl
+                .items
+                .iter()
+                .filter_map(|s| pl.contains(&s.snippet.title).then_some(s.id.clone()))
+                .collect(),
+        );
+
+        // if nextPageToken is present, continue the loop
+        match local_pl.next_page_token {
+            None => break,
+            Some(a) => page_token = a.clone(),
+        }
+    }
+
+    debug!("Playlists IDs: {:?}", &playlists);
+    Ok(playlists)
+}
+
+/// This function adds the video id to the corresponding named playlist(s)
+pub async fn add_video_to_playlists(
+    config: &YoutubeConfig,
+    v: &str,
+    pl: &[String],
+) -> Result<(), Box<dyn Error>> {
+    let access_token = refresh_token(config).await?;
+    let playlists_ids = get_playlist_ids(config, pl).await?;
+
+    for pl_id in playlists_ids {
+        let yt_pl_upload_params = YoutubePlaylistItemsParams {
+            snippet: YoutubePlaylistItemsParamsSnippet {
+                playlist_id: pl_id.clone(),
+                resource_id: YoutubePlaylistItemsParamsSnippetResourceId {
+                    video_id: v.to_string(),
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+        };
+
+        let client = Client::new();
+        let res = client
+            .post("https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet")
+            .header("Authorization", format!("Bearer {}", access_token))
+            .json(&yt_pl_upload_params)
+            .send()
+            .await?;
+
+        if !res.status().is_success() {
+            return Err(TootubeError::new(&format!(
+                "Something went wrong when trying to add the video to a playlist: {}",
+                res.text().await?
+            ))
+            .into());
+        }
+    }
+
+    Ok(())
+}
+
+/// This function creates a resumable YT upload, putting all the parameters in
 pub async fn create_resumable_upload(
     config: &YoutubeConfig,
     vid: &PeerTubeVideo,
@@ -129,9 +319,9 @@ pub async fn create_resumable_upload(
             }
         },
     };
+    debug!("YT upload params: {:?}", &upload_params);
 
     let client = Client::new();
-
     let res = client.post("https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet%2Cstatus")
         .header("Authorization", format!("Bearer {}", access_token))
         .json(&upload_params)
@@ -149,6 +339,7 @@ pub async fn create_resumable_upload(
     }
 }
 
+/// This takes the PT stream for download, connects it to YT stream for upload
 pub async fn now_kiss<'a>(
     stream: impl Stream<Item = Result<Bytes, reqwest::Error>>
         + std::marker::Send
@@ -156,13 +347,12 @@ pub async fn now_kiss<'a>(
         + 'a + 'static,
     r_url: &'a str,
     config: &'a YoutubeConfig,
-) -> Result<(), Box<dyn Error>> {
+) -> Result<String, Box<dyn Error>> {
     // Get access token
     let access_token = refresh_token(config).await?;
 
     // Create client
     let client = Client::new();
-
     let res = client
         .put(r_url)
         .header("Authorization", format!("Bearer {}", access_token))
@@ -171,7 +361,8 @@ pub async fn now_kiss<'a>(
         .await?;
 
     if res.status().is_success() {
-        Ok(())
+        let yt_videos: YoutubeVideos = res.json().await?;
+        Ok(yt_videos.id)
     } else {
         Err(TootubeError::new(&format!("Cannot upload video: {:?}", res.text().await?)).into())
     }