5 Commits

Author SHA1 Message Date
VC
1bdd747202 Merge branch 'feat_cargo_update+style' into 'master'
Feat cargo update+style

See merge request veretcle/tootube!6
2023-10-11 06:40:34 +00:00
VC
0640c4c263 style: regroup 'use' 2023-10-10 21:13:40 +02:00
VC
59ad8994b1 chore: cargo update 2023-10-10 21:09:10 +02:00
VC
f1d05e9785 Merge branch 'rev0-2' into 'master'
feat: async + stream

See merge request veretcle/tootube!5
2023-10-09 12:09:38 +00:00
VC
0451543d6d feat: async + stream 2023-10-09 13:07:03 +02:00
6 changed files with 225 additions and 132 deletions

198
Cargo.lock generated
View File

@@ -19,18 +19,18 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab"
checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
dependencies = [
"memchr",
]
[[package]]
name = "anstream"
version = "0.6.1"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6cd65a4b849ace0b7f6daeebcc1a1d111282227ca745458c61dbf670e52a597"
checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -66,9 +66,9 @@ dependencies = [
[[package]]
name = "anstyle-wincon"
version = "3.0.0"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0238ca56c96dfa37bdf7c373c8886dd591322500aceeeccdb2216fe06dc2f796"
checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
dependencies = [
"anstyle",
"windows-sys",
@@ -213,25 +213,14 @@ dependencies = [
[[package]]
name = "errno"
version = "0.3.3"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "fastrand"
version = "2.0.1"
@@ -289,6 +278,17 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
[[package]]
name = "futures-macro"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.28"
@@ -309,6 +309,8 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
dependencies = [
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
@@ -349,9 +351,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hermit-abi"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "http"
@@ -490,15 +492,25 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.148"
version = "0.2.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
[[package]]
name = "linux-raw-sys"
version = "0.4.7"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
[[package]]
name = "lock_api"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
@@ -508,9 +520,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "memchr"
version = "2.6.3"
version = "2.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
[[package]]
name = "mime"
@@ -625,6 +637,29 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "percent-encoding"
version = "2.3.0"
@@ -651,9 +686,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "proc-macro2"
version = "1.0.67"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328"
checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
dependencies = [
"unicode-ident",
]
@@ -678,9 +713,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.9.6"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff"
checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87"
dependencies = [
"aho-corasick",
"memchr",
@@ -690,9 +725,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.9"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b"
dependencies = [
"aho-corasick",
"memchr",
@@ -701,15 +736,15 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.7.5"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d"
[[package]]
name = "reqwest"
version = "0.11.20"
version = "0.11.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
dependencies = [
"base64",
"bytes",
@@ -732,12 +767,15 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
"system-configuration",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"winreg",
]
@@ -750,9 +788,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustix"
version = "0.38.14"
version = "0.38.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f"
checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c"
dependencies = [
"bitflags 2.4.0",
"errno",
@@ -776,6 +814,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "security-framework"
version = "2.9.2"
@@ -842,6 +886,15 @@ dependencies = [
"serde",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
dependencies = [
"libc",
]
[[package]]
name = "slab"
version = "0.4.9"
@@ -851,6 +904,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
[[package]]
name = "socket2"
version = "0.4.9"
@@ -879,15 +938,36 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "2.0.37"
version = "2.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8"
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "tempfile"
version = "3.8.0"
@@ -927,20 +1007,34 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.32.0"
version = "1.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"num_cpus",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.5.4",
"tokio-macros",
"windows-sys",
]
[[package]]
name = "tokio-macros"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
@@ -976,13 +1070,16 @@ dependencies = [
[[package]]
name = "tootube"
version = "0.2.0"
version = "0.3.0"
dependencies = [
"bytes",
"clap",
"env_logger",
"futures-core",
"log",
"reqwest",
"serde",
"tokio",
"toml",
]
@@ -1143,6 +1240,19 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "wasm-streams"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.64"

View File

@@ -1,18 +1,21 @@
[package]
name = "tootube"
authors = ["VC <veretcle+framagit@mateu.be>"]
version = "0.2.0"
version = "0.3.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
reqwest = { version = "^0.11", features = ["blocking", "json"] }
reqwest = { version = "^0.11", features = ["json", "stream"] }
tokio = { version = "^1", features = ["full"] }
clap = "^4"
serde = { version = "1.0", features = ["derive"] }
toml = "^0.5"
log = "^0.4"
env_logger = "^0.10"
futures-core = "0.3.28"
bytes = "1.5.0"
[profile.release]
strip = true

View File

@@ -1,6 +1,5 @@
use std::fs::read_to_string;
use serde::Deserialize;
use std::fs::read_to_string;
// General configuration Struct
#[derive(Debug, Deserialize)]

View File

@@ -1,13 +1,8 @@
use std::{
error::Error,
fs::create_dir_all,
fs::{remove_file, File},
};
use reqwest::Url;
use bytes::Bytes;
use futures_core::stream::Stream;
use std::error::Error;
mod error;
use error::TootubeError;
mod config;
pub use config::parse_toml;
@@ -17,58 +12,34 @@ mod peertube;
use peertube::{get_latest_video, get_max_resolution_dl};
mod youtube;
use youtube::{create_resumable_upload, upload_video};
use youtube::{create_resumable_upload, now_kiss};
const TMP_DIR: &str = "/tmp/tootube";
fn dl_video(u: &str) -> Result<String, Box<dyn Error>> {
// create dir
create_dir_all(TMP_DIR)?;
// get file
let mut response = reqwest::blocking::get(u)?;
// create local file
let url = Url::parse(u)?;
let dest_filename = url
.path_segments()
.ok_or_else(|| {
TootubeError::new(&format!(
"Cannot determine the destination filename for {u}"
))
})?
.last()
.ok_or_else(|| {
TootubeError::new(&format!(
"Cannot determine the destination filename for {u}"
))
})?;
let dest_filepath = format!("{TMP_DIR}/{dest_filename}");
let mut dest_file = File::create(&dest_filepath)?;
response.copy_to(&mut dest_file)?;
Ok(dest_filepath)
async fn get_dl_video_stream(
u: &str,
) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>, Box<dyn Error>> {
Ok(reqwest::get(u).await?.bytes_stream())
}
pub fn run(config: Config) {
#[tokio::main]
pub async fn run(config: Config) {
// Get the latest video object
let latest_vid = get_latest_video(&config.peertube.base_url).unwrap_or_else(|e| {
panic!("Cannot retrieve the latest video, something must have gone terribly wrong: {e}")
});
let latest_vid = get_latest_video(&config.peertube.base_url)
.await
.unwrap_or_else(|e| {
panic!("Cannot retrieve the latest video, something must have gone terribly wrong: {e}")
});
let dl_url = get_max_resolution_dl(latest_vid.streaming_playlists.as_ref().unwrap());
let local_path = dl_video(&dl_url)
let pt_stream = get_dl_video_stream(&dl_url)
.await
.unwrap_or_else(|e| panic!("Cannot download video at URL {}: {}", dl_url, e));
let resumable_upload_url = create_resumable_upload(&config.youtube, &latest_vid)
.await
.unwrap_or_else(|e| panic!("Cannot retrieve the uploads resumable id: {e}"));
upload_video(&local_path, &resumable_upload_url, &config.youtube)
now_kiss(pt_stream, &resumable_upload_url, &config.youtube)
.await
.unwrap_or_else(|e| panic!("Cannot resume upload!: {e}"));
remove_file(&local_path).unwrap_or_else(|e| panic!("Cannot delete file {}: {}", local_path, e));
}

View File

@@ -36,11 +36,13 @@ pub struct PeerTubeVideoStreamingPlaylistsFilesResolution {
}
/// This gets the last video uploaded to the PeerTube server
pub fn get_latest_video(u: &str) -> Result<PeerTubeVideo, Box<dyn Error>> {
let body = reqwest::blocking::get(format!("{}/api/v1/videos?count=1&sort=-publishedAt", u))?
.json::<PeerTubeVideos>()?;
pub async fn get_latest_video(u: &str) -> Result<PeerTubeVideo, Box<dyn Error>> {
let body = reqwest::get(format!("{}/api/v1/videos?count=1&sort=-publishedAt", u))
.await?
.json::<PeerTubeVideos>()
.await?;
let vid = get_video_detail(u, &body.data[0].uuid)?;
let vid = get_video_detail(u, &body.data[0].uuid).await?;
Ok(vid)
}
@@ -61,9 +63,11 @@ pub fn get_max_resolution_dl(p: &[PeerTubeVideoStreamingPlaylists]) -> String {
}
/// This gets all the crispy details about one particular video
fn get_video_detail(u: &str, v: &str) -> Result<PeerTubeVideo, Box<dyn Error>> {
let body =
reqwest::blocking::get(format!("{}/api/v1/videos/{}", u, v))?.json::<PeerTubeVideo>()?;
async fn get_video_detail(u: &str, v: &str) -> Result<PeerTubeVideo, Box<dyn Error>> {
let body = reqwest::get(format!("{}/api/v1/videos/{}", u, v))
.await?
.json::<PeerTubeVideo>()
.await?;
Ok(body)
}

View File

@@ -1,8 +1,12 @@
use crate::{config::YoutubeConfig, error::TootubeError, peertube::PeerTubeVideo};
use bytes::Bytes;
use futures_core::stream::Stream;
use reqwest::{Body, Client};
use serde::{Deserialize, Serialize};
use std::{error::Error, fs::File, sync::Mutex};
use std::error::Error;
use tokio::sync::OnceCell;
static ACCESS_TOKEN: Mutex<String> = Mutex::new(String::new());
static ACCESS_TOKEN: OnceCell<String> = OnceCell::const_new();
#[derive(Serialize, Debug)]
struct RefreshTokenRequest {
@@ -79,9 +83,9 @@ impl Default for YoutubeUploadParamsStatus {
}
/// Ensures that Token has been refreshed and that it is unique
fn refresh_token(config: &YoutubeConfig) -> Result<String, Box<dyn Error>> {
if let Ok(mut unlocked_access_token) = ACCESS_TOKEN.lock() {
if unlocked_access_token.is_empty() {
async fn refresh_token(config: &YoutubeConfig) -> Result<String, reqwest::Error> {
ACCESS_TOKEN
.get_or_try_init(|| async {
let refresh_token = RefreshTokenRequest {
refresh_token: config.refresh_token.clone(),
client_id: config.client_id.clone(),
@@ -89,26 +93,26 @@ fn refresh_token(config: &YoutubeConfig) -> Result<String, Box<dyn Error>> {
..Default::default()
};
let client = reqwest::blocking::Client::new();
let client = Client::new();
let res = client
.post("https://accounts.google.com/o/oauth2/token")
.json(&refresh_token)
.send()?;
.send()
.await?;
let access_token: AccessTokenResponse = res.json()?;
let access_token: AccessTokenResponse = res.json().await?;
*unlocked_access_token = access_token.access_token.clone();
}
}
Ok(ACCESS_TOKEN.lock().unwrap().to_string())
Ok(access_token.access_token)
})
.await
.cloned()
}
pub fn create_resumable_upload(
pub async fn create_resumable_upload(
config: &YoutubeConfig,
vid: &PeerTubeVideo,
) -> Result<String, Box<dyn Error>> {
let access_token = refresh_token(config)?;
let access_token = refresh_token(config).await?;
let upload_params = YoutubeUploadParams {
snippet: {
@@ -126,12 +130,12 @@ pub fn create_resumable_upload(
},
};
let client = reqwest::blocking::Client::new();
let client = Client::new();
let res = client.post("https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet%2Cstatus")
.header("Authorization", format!("Bearer {}", access_token))
.json(&upload_params)
.send()?;
.send().await?;
if res.status().is_success() {
Ok(res
@@ -145,28 +149,30 @@ pub fn create_resumable_upload(
}
}
pub fn upload_video(
f_path: &str,
r_url: &str,
config: &YoutubeConfig,
pub async fn now_kiss<'a>(
stream: impl Stream<Item = Result<Bytes, reqwest::Error>>
+ std::marker::Send
+ std::marker::Sync
+ 'a + 'static,
r_url: &'a str,
config: &'a YoutubeConfig,
) -> Result<(), Box<dyn Error>> {
// Get access token
let access_token = refresh_token(config)?;
let access_token = refresh_token(config).await?;
// Create client
let client = reqwest::blocking::Client::new();
let file = File::open(f_path)?;
let client = Client::new();
let res = client
.put(r_url)
.header("Authorization", format!("Bearer {}", access_token))
.body(file)
.send()?;
.body(Body::wrap_stream(stream))
.send()
.await?;
if res.status().is_success() {
Ok(())
} else {
Err(TootubeError::new(&format!("Cannot upload video: {:?}", res.text()?)).into())
Err(TootubeError::new(&format!("Cannot upload video: {:?}", res.text().await?)).into())
}
}