mirror of
https://framagit.org/veretcle/scootaloo.git
synced 2025-07-20 17:11:19 +02:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
da808b0051 | ||
|
|
5a4dd5cb99 | ||
|
|
5b04bd27b9 | ||
|
|
c52fc52d23 | ||
|
|
09ed837a1b | ||
|
|
d4db2933ae | ||
|
|
2e052ebf6a |
@@ -3,8 +3,11 @@ stages:
|
||||
|
||||
rust-latest:
|
||||
stage: build
|
||||
artifacts:
|
||||
paths:
|
||||
- target/release/scootaloo
|
||||
image: rust:latest
|
||||
script:
|
||||
- cargo build --verbose
|
||||
- cargo test --verbose
|
||||
- cargo build --release --verbose
|
||||
- strip target/release/${CI_PROJECT_NAME}
|
||||
|
||||
|
||||
20
CHANGELOG
20
CHANGELOG
@@ -1,3 +1,23 @@
|
||||
# v0.3.3
|
||||
|
||||
* optimizing the size of the final executable (now ⩽ 6MiB)
|
||||
|
||||
# v0.3.2
|
||||
|
||||
* 100% async version
|
||||
* now media are download in parallel thanks to async
|
||||
* log are introduced into code for your viewing pleasure
|
||||
|
||||
# v0.2.3
|
||||
|
||||
* using the async version of `reqwest`
|
||||
* introducing async functions and make `tokio` the de facto executor for everything async
|
||||
|
||||
# v0.2.1
|
||||
|
||||
* using `tokio-compat` to avoid having 3 different versions of `tokio` in the same executable
|
||||
* encapsulating async calls inside blocking tokio runtime calls
|
||||
|
||||
# v0.1.8
|
||||
|
||||
* fix #1: mentions are treated like decoded urls (this is not really needed to push it this far but it would be easier in case you want to modify it)
|
||||
|
||||
339
Cargo.lock
generated
339
Cargo.lock
generated
@@ -94,12 +94,6 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.13.0"
|
||||
@@ -265,6 +259,17 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colored"
|
||||
version = "1.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"lazy_static",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.12.0"
|
||||
@@ -373,9 +378,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-mac"
|
||||
version = "0.8.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
|
||||
checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6"
|
||||
dependencies = [
|
||||
"generic-array 0.14.4",
|
||||
"subtle",
|
||||
@@ -431,27 +436,26 @@ checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3"
|
||||
[[package]]
|
||||
name = "egg-mode"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f877bc908325f50163ff1670a4733eabf87942511ccfd907fbef4e239c3f8aa"
|
||||
source = "git+https://github.com/egg-mode-rs/egg-mode?rev=6b81073eba9c3b123ca0e80bdb5ef61d1758f131#6b81073eba9c3b123ca0e80bdb5ef61d1758f131"
|
||||
dependencies = [
|
||||
"base64 0.12.3",
|
||||
"base64 0.13.0",
|
||||
"chrono",
|
||||
"derive_more",
|
||||
"futures 0.3.5",
|
||||
"hmac",
|
||||
"hyper 0.13.2",
|
||||
"hyper-tls 0.4.1",
|
||||
"hyper 0.14.4",
|
||||
"hyper-tls 0.5.0",
|
||||
"lazy_static",
|
||||
"mime",
|
||||
"native-tls",
|
||||
"percent-encoding 2.1.0",
|
||||
"rand 0.7.3",
|
||||
"rand 0.8.3",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha-1 0.9.4",
|
||||
"thiserror",
|
||||
"tokio 0.2.25",
|
||||
"tokio 1.3.0",
|
||||
"url 2.2.1",
|
||||
]
|
||||
|
||||
@@ -735,7 +739,18 @@ checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.9",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasi 0.9.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"libc",
|
||||
"wasi 0.10.2+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -762,25 +777,6 @@ dependencies = [
|
||||
"tokio-io",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9433d71e471c1736fd5a61b671fc0b148d7a2992f666c958d03cd8feb3b88d1"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http 0.2.0",
|
||||
"indexmap",
|
||||
"log",
|
||||
"slab",
|
||||
"tokio 0.2.25",
|
||||
"tokio-util 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.1"
|
||||
@@ -796,7 +792,7 @@ dependencies = [
|
||||
"indexmap",
|
||||
"slab",
|
||||
"tokio 1.3.0",
|
||||
"tokio-util 0.6.4",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -817,9 +813,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.8.1"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840"
|
||||
checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
|
||||
dependencies = [
|
||||
"crypto-mac",
|
||||
"digest 0.9.0",
|
||||
@@ -865,16 +861,6 @@ dependencies = [
|
||||
"tokio-buf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"http 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.4.0"
|
||||
@@ -927,30 +913,6 @@ dependencies = [
|
||||
"want 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa1c527bbc634be72aa7ba31e4e4def9bbb020f5416916279b7c705cd838893e"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.2.1",
|
||||
"http 0.2.0",
|
||||
"http-body 0.3.1",
|
||||
"httparse",
|
||||
"itoa",
|
||||
"log",
|
||||
"net2",
|
||||
"pin-project 0.4.8",
|
||||
"time",
|
||||
"tokio 0.2.25",
|
||||
"tower-service",
|
||||
"want 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.14.4"
|
||||
@@ -1005,19 +967,6 @@ dependencies = [
|
||||
"tokio-io",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"hyper 0.13.2",
|
||||
"native-tls",
|
||||
"tokio 0.2.25",
|
||||
"tokio-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.5.0"
|
||||
@@ -1072,6 +1021,15 @@ dependencies = [
|
||||
"bytes 0.5.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iovec"
|
||||
version = "0.1.4"
|
||||
@@ -1150,6 +1108,15 @@ dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.8"
|
||||
@@ -1389,11 +1356,22 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
"lock_api 0.3.3",
|
||||
"parking_lot_core 0.6.2",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api 0.4.3",
|
||||
"parking_lot_core 0.8.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.6.2"
|
||||
@@ -1409,6 +1387,20 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec 1.2.0",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "1.0.1"
|
||||
@@ -1499,12 +1491,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.6"
|
||||
@@ -1525,9 +1511,9 @@ checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.6"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
|
||||
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-hack"
|
||||
@@ -1619,13 +1605,25 @@ version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.1.14",
|
||||
"libc",
|
||||
"rand_chacha 0.2.1",
|
||||
"rand_core 0.5.1",
|
||||
"rand_hc 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.0",
|
||||
"rand_core 0.6.2",
|
||||
"rand_hc 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.1.1"
|
||||
@@ -1646,6 +1644,16 @@ dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.3.1"
|
||||
@@ -1667,7 +1675,16 @@ version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.1.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
|
||||
dependencies = [
|
||||
"getrandom 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1688,6 +1705,15 @@ dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
|
||||
dependencies = [
|
||||
"rand_core 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_isaac"
|
||||
version = "0.1.1"
|
||||
@@ -1839,7 +1865,7 @@ dependencies = [
|
||||
"mime",
|
||||
"native-tls",
|
||||
"percent-encoding 2.1.0",
|
||||
"pin-project-lite 0.2.6",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
"serde_urlencoded 0.7.0",
|
||||
"tokio 1.3.0",
|
||||
@@ -1899,16 +1925,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "scootaloo"
|
||||
version = "0.2.1"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"egg-mode",
|
||||
"elefren",
|
||||
"htmlescape",
|
||||
"log",
|
||||
"reqwest 0.11.2",
|
||||
"serde",
|
||||
"simple_logger",
|
||||
"tokio 1.3.0",
|
||||
"tokio-compat-02",
|
||||
"toml",
|
||||
]
|
||||
|
||||
@@ -2061,6 +2088,28 @@ dependencies = [
|
||||
"opaque-debug 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simple_logger"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd57f17c093ead1d4a1499dc9acaafdd71240908d64775465543b8d9a9f1d198"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"chrono",
|
||||
"colored",
|
||||
"log",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.2.3"
|
||||
@@ -2257,24 +2306,6 @@ dependencies = [
|
||||
"tokio-timer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "0.2.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"fnv",
|
||||
"iovec",
|
||||
"lazy_static",
|
||||
"memchr",
|
||||
"mio 0.6.23",
|
||||
"num_cpus",
|
||||
"pin-project-lite 0.1.4",
|
||||
"slab",
|
||||
"tokio-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.3.0"
|
||||
@@ -2287,7 +2318,12 @@ dependencies = [
|
||||
"memchr",
|
||||
"mio 0.7.9",
|
||||
"num_cpus",
|
||||
"pin-project-lite 0.2.6",
|
||||
"once_cell",
|
||||
"parking_lot 0.11.1",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"tokio-macros",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2301,20 +2337,6 @@ dependencies = [
|
||||
"futures 0.1.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-compat-02"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7d4237822b7be8fff0a7a27927462fad435dcb6650f95cea9e946bf6bdc7e07"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"once_cell",
|
||||
"pin-project-lite 0.2.6",
|
||||
"tokio 0.2.25",
|
||||
"tokio 1.3.0",
|
||||
"tokio-stream",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-current-thread"
|
||||
version = "0.1.7"
|
||||
@@ -2348,9 +2370,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "0.2.6"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a"
|
||||
checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -2379,24 +2401,13 @@ dependencies = [
|
||||
"log",
|
||||
"mio 0.6.23",
|
||||
"num_cpus",
|
||||
"parking_lot",
|
||||
"parking_lot 0.9.0",
|
||||
"slab",
|
||||
"tokio-executor",
|
||||
"tokio-io",
|
||||
"tokio-sync",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite 0.2.6",
|
||||
"tokio 1.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-sync"
|
||||
version = "0.1.8"
|
||||
@@ -2450,30 +2461,6 @@ dependencies = [
|
||||
"tokio-executor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tls"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio 0.2.25",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"log",
|
||||
"pin-project-lite 0.1.4",
|
||||
"tokio 0.2.25",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.6.4"
|
||||
@@ -2484,7 +2471,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"log",
|
||||
"pin-project-lite 0.2.6",
|
||||
"pin-project-lite",
|
||||
"tokio 1.3.0",
|
||||
]
|
||||
|
||||
@@ -2510,7 +2497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"pin-project-lite 0.2.6",
|
||||
"pin-project-lite",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
@@ -2703,6 +2690,12 @@ version = "0.9.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.71"
|
||||
|
||||
25
Cargo.toml
25
Cargo.toml
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "scootaloo"
|
||||
version = "0.2.1"
|
||||
version = "0.3.3"
|
||||
authors = ["VC <veretcle+framagit@mateu.be>"]
|
||||
edition = "2018"
|
||||
|
||||
@@ -8,18 +8,19 @@ edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
|
||||
toml = "^0.5"
|
||||
|
||||
clap = "^2.33"
|
||||
|
||||
tokio = { version = "1", features = ["rt-multi-thread"]}
|
||||
tokio-compat-02 = "0.2"
|
||||
|
||||
egg-mode = "^0.15"
|
||||
|
||||
egg-mode = { git = "https://github.com/egg-mode-rs/egg-mode", rev = "6b81073eba9c3b123ca0e80bdb5ef61d1758f131" }
|
||||
elefren = "^0.22"
|
||||
|
||||
reqwest = { version="^0.11", features = ["blocking"] }
|
||||
|
||||
tokio = { version = "1", features = ["full"]}
|
||||
reqwest = "^0.11"
|
||||
htmlescape = "^0.3"
|
||||
log = "^0.4"
|
||||
simple_logger = "^1.11"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 's' # Optimize for size.
|
||||
lto = true # Link Time Optimization (LTO)
|
||||
codegen-units = 1 # Set this to 1 to allow for maximum size reduction optimizations:
|
||||
panic = 'abort' # removes the need for this extra unwinding code.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ A Twitter to Mastodon copy bot written in Rust
|
||||
It:
|
||||
* copies the content (text) of the original Tweet
|
||||
* dereferences the links
|
||||
* gets every attach media (photo, video or gif)
|
||||
* gets every attached media (photo, video or gif)
|
||||
|
||||
If any of the last steps failed, the Toot gets published with the exact same text as the Tweet.
|
||||
|
||||
@@ -74,4 +74,3 @@ echo -n '8189881949849' > last_tweet
|
||||
|
||||
**This file should only contain the last tweet ID without any other char (no EOL or new line).**
|
||||
|
||||
Oh and everything is sync (and not async) so this does not run at a blazing speed…
|
||||
|
||||
182
src/lib.rs
182
src/lib.rs
@@ -1,18 +1,14 @@
|
||||
// std
|
||||
use std::{
|
||||
path::Path,
|
||||
borrow::Cow,
|
||||
collections::HashMap,
|
||||
io::{stdin, copy},
|
||||
io::stdin,
|
||||
fmt,
|
||||
fs::{read_to_string, write, create_dir_all, File, remove_file},
|
||||
fs::{read_to_string, write},
|
||||
error::Error,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
//tokio
|
||||
use tokio::runtime::Runtime;
|
||||
use tokio_compat_02::FutureExt;
|
||||
|
||||
// toml
|
||||
use serde::Deserialize;
|
||||
|
||||
@@ -37,34 +33,45 @@ use elefren::{
|
||||
};
|
||||
|
||||
// reqwest
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::Url;
|
||||
|
||||
// tokio
|
||||
use tokio::{
|
||||
io::copy,
|
||||
fs::{File, create_dir_all, remove_file},
|
||||
sync::mpsc,
|
||||
};
|
||||
|
||||
// htmlescape
|
||||
use htmlescape::decode_html;
|
||||
|
||||
// log
|
||||
use log::{info, warn, error, debug};
|
||||
|
||||
/**********
|
||||
* Generic usage functions
|
||||
***********/
|
||||
/*
|
||||
* Those functions are related to the Twitter side of things
|
||||
*/
|
||||
/// Read last tweet id from a file
|
||||
/// Reads last tweet id from a file
|
||||
fn read_state(s: &str) -> Option<u64> {
|
||||
let state = read_to_string(s);
|
||||
|
||||
if let Ok(s) = state {
|
||||
debug!("Last Tweet ID (from file): {}", &s);
|
||||
return s.parse::<u64>().ok();
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Write last treated tweet id to a file
|
||||
/// Writes last treated tweet id to a file
|
||||
fn write_state(f: &str, s: u64) -> Result<(), std::io::Error> {
|
||||
write(f, format!("{}", s))
|
||||
}
|
||||
|
||||
/// Get twitter oauth2 token
|
||||
/// Gets Twitter oauth2 token
|
||||
fn get_oauth2_token(config: &Config) -> Token {
|
||||
let con_token = KeyPair::new(String::from(&config.twitter.consumer_key), String::from(&config.twitter.consumer_secret));
|
||||
let access_token = KeyPair::new(String::from(&config.twitter.access_key), String::from(&config.twitter.access_secret));
|
||||
@@ -75,19 +82,18 @@ fn get_oauth2_token(config: &Config) -> Token {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get twitter user timeline
|
||||
fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
|
||||
/// Gets Twitter user timeline
|
||||
async fn get_user_timeline(config: &Config, token: Token, lid: Option<u64>) -> Result<Vec<Tweet>, Box<dyn Error>> {
|
||||
// fix the page size to 200 as it is the maximum Twitter authorizes
|
||||
let rt = Runtime::new()?;
|
||||
let (_timeline, feed) = rt.block_on(user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
|
||||
let (_, feed) = user_timeline(UserID::from(String::from(&config.twitter.username)), true, false, &token)
|
||||
.with_page_size(200)
|
||||
.older(lid)
|
||||
.compat())?;
|
||||
.await?;
|
||||
|
||||
Ok(feed.to_vec())
|
||||
}
|
||||
|
||||
/// decode urls from UrlEntities
|
||||
/// Decodes urls from UrlEntities
|
||||
fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_urls = HashMap::new();
|
||||
|
||||
@@ -101,6 +107,8 @@ fn decode_urls(urls: &Vec<UrlEntity>) -> HashMap<String, String> {
|
||||
decoded_urls
|
||||
}
|
||||
|
||||
/// Decodes the Twitter mention to something that will make sense once Twitter has joined the
|
||||
/// Fediverse
|
||||
fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
|
||||
let mut decoded_mentions = HashMap::new();
|
||||
|
||||
@@ -111,18 +119,18 @@ fn twitter_mentions(ums: &Vec<MentionEntity>) -> HashMap<String, String> {
|
||||
decoded_mentions
|
||||
}
|
||||
|
||||
/// Retrieve a single media from a tweet and store it in a temporary file
|
||||
fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
/// Retrieves a single media from a tweet and store it in a temporary file
|
||||
async fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
match m.media_type {
|
||||
MediaType::Photo => {
|
||||
return cache_media(&m.media_url_https, t);
|
||||
return cache_media(&m.media_url_https, t).await;
|
||||
},
|
||||
_ => {
|
||||
match &m.video_info {
|
||||
Some(v) => {
|
||||
for variant in &v.variants {
|
||||
if variant.content_type == "video/mp4" {
|
||||
return cache_media(&variant.url, t);
|
||||
return cache_media(&variant.url, t).await;
|
||||
}
|
||||
}
|
||||
return Err(Box::new(ScootalooError::new(format!("Media Type for {} is video but no mp4 file URL is available", &m.url).as_str())));
|
||||
@@ -138,7 +146,7 @@ fn get_tweet_media(m: &MediaEntity, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
/*
|
||||
* Those functions are related to the Mastodon side of things
|
||||
*/
|
||||
/// Get Mastodon Data
|
||||
/// Gets Mastodon Data
|
||||
fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
|
||||
let data = Data {
|
||||
base: Cow::from(String::from(&masto.base)),
|
||||
@@ -151,7 +159,7 @@ fn get_mastodon_token(masto: &MastodonConfig) -> Mastodon {
|
||||
Mastodon::from(data)
|
||||
}
|
||||
|
||||
/// build toot text from tweet
|
||||
/// Builds toot text from tweet
|
||||
fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
|
||||
let mut toot = String::from(&tweet.text);
|
||||
|
||||
@@ -177,35 +185,40 @@ fn build_basic_status(tweet: &Tweet) -> Result<String, Box<dyn Error>> {
|
||||
/*
|
||||
* Generic private functions
|
||||
*/
|
||||
fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
|
||||
/// Gets and caches Twitter Media inside the determined temp dir
|
||||
async fn cache_media(u: &str, t: &str) -> Result<String, Box<dyn Error>> {
|
||||
// create dir
|
||||
if !Path::new(t).is_dir() {
|
||||
create_dir_all(t)?;
|
||||
}
|
||||
create_dir_all(t).await?;
|
||||
|
||||
// get file
|
||||
let client = Client::new();
|
||||
let mut response = client.get(u).send()?;
|
||||
let mut response = reqwest::get(u).await?;
|
||||
|
||||
// create local file
|
||||
let dest_filename = match response.url()
|
||||
.path_segments()
|
||||
.and_then(|segments| segments.last()) {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
return Err(Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())));
|
||||
},
|
||||
};
|
||||
let url = Url::parse(u)?;
|
||||
let dest_filename = url.path_segments().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?
|
||||
.last().ok_or_else(|| Box::new(ScootalooError::new(format!("Cannot determine the destination filename for {}", u).as_str())))?;
|
||||
|
||||
let dest_filepath = format!("{}/{}", t, dest_filename);
|
||||
|
||||
let mut dest_file = File::create(&dest_filepath)?;
|
||||
let mut dest_file = File::create(&dest_filepath).await?;
|
||||
|
||||
copy(&mut response, &mut dest_file)?;
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
copy(&mut &*chunk, &mut dest_file).await?;
|
||||
}
|
||||
|
||||
Ok(dest_filepath)
|
||||
}
|
||||
|
||||
/**********
|
||||
* This is the struct that holds the Mastodon Media ID and the Twitter Media URL at the same Time
|
||||
**********/
|
||||
#[derive(Debug)]
|
||||
struct ScootalooSpawnResponse {
|
||||
mastodon_media_id: String,
|
||||
twitter_media_url: String,
|
||||
}
|
||||
|
||||
/**********
|
||||
* local error handler
|
||||
**********/
|
||||
@@ -287,7 +300,7 @@ pub fn parse_toml(toml_file: &str) -> Config {
|
||||
|
||||
/// Generic register function
|
||||
/// As this function is supposed to be run only once, it will panic for every error it encounters
|
||||
/// Most of this function is a direct copy/paste of the official `mammut` crate
|
||||
/// Most of this function is a direct copy/paste of the official `elefren` crate
|
||||
pub fn register(host: &str) {
|
||||
let mut builder = App::builder();
|
||||
builder.client_name(Cow::from(String::from(env!("CARGO_PKG_NAME"))))
|
||||
@@ -315,7 +328,8 @@ pub fn register(host: &str) {
|
||||
}
|
||||
|
||||
/// This is where the magic happens
|
||||
pub fn run(config: Config) {
|
||||
#[tokio::main]
|
||||
pub async fn run(config: Config) {
|
||||
// retrieve the last tweet ID for the username
|
||||
let last_tweet_id = read_state(&config.scootaloo.last_tweet_path);
|
||||
|
||||
@@ -323,16 +337,18 @@ pub fn run(config: Config) {
|
||||
let token = get_oauth2_token(&config);
|
||||
|
||||
// get Mastodon instance
|
||||
let mastodon = get_mastodon_token(&config.mastodon);
|
||||
let mastodon = Arc::new(Mutex::new(get_mastodon_token(&config.mastodon)));
|
||||
|
||||
// get user timeline feed (Vec<tweet>)
|
||||
let mut feed = get_user_timeline(&config, token, last_tweet_id).unwrap_or_else(|e|
|
||||
let mut feed = get_user_timeline(&config, token, last_tweet_id)
|
||||
.await
|
||||
.unwrap_or_else(|e|
|
||||
panic!("Something went wrong when trying to retrieve {}’s timeline: {}", &config.twitter.username, e)
|
||||
);
|
||||
|
||||
// empty feed -> exiting
|
||||
if feed.is_empty() {
|
||||
println!("Nothing to retrieve since last time, exiting…");
|
||||
info!("Nothing to retrieve since last time, exiting…");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -340,10 +356,12 @@ pub fn run(config: Config) {
|
||||
feed.reverse();
|
||||
|
||||
for tweet in &feed {
|
||||
debug!("Treating Tweet {} inside feed", tweet.id);
|
||||
// determine if the tweet is part of a thread (response to self) or a standard response
|
||||
if let Some(r) = &tweet.in_reply_to_screen_name {
|
||||
if &r.to_lowercase() != &config.twitter.username.to_lowercase() {
|
||||
// we are responding not threading
|
||||
info!("Tweet is a direct response, skipping");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -352,7 +370,7 @@ pub fn run(config: Config) {
|
||||
let mut status_text = match build_basic_status(tweet) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
println!("Could not create status from tweet {}: {}", tweet.id ,e);
|
||||
error!("Could not create status from tweet {}: {}", tweet.id ,e);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
@@ -361,35 +379,64 @@ pub fn run(config: Config) {
|
||||
|
||||
// reupload the attachments if any
|
||||
if let Some(m) = &tweet.extended_entities {
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
|
||||
for media in &m.media {
|
||||
let local_tweet_media_path = match get_tweet_media(&media, &config.scootaloo.cache_path) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
println!("Cannot get tweet media for {}: {}", &media.url, e);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
// creating a new tx for this initial loop
|
||||
let tx = tx.clone();
|
||||
// creating a new mastodon from the original mutex
|
||||
let mastodon = mastodon.clone();
|
||||
// unfortunately for this to be thread safe, we need to clone a lot of structures
|
||||
let media = media.clone();
|
||||
let cache_path = config.scootaloo.cache_path.clone();
|
||||
|
||||
let mastodon_media_ids = match mastodon.media(Cow::from(String::from(&local_tweet_media_path))) {
|
||||
Ok(m) => {
|
||||
remove_file(&local_tweet_media_path).unwrap_or_else(|e|
|
||||
println!("Attachment for {} has been upload, but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
|
||||
);
|
||||
m.id
|
||||
},
|
||||
Err(e) => {
|
||||
println!("Cannot attach media {} to Mastodon Instance: {}", &local_tweet_media_path, e);
|
||||
continue;
|
||||
tokio::spawn(async move {
|
||||
debug!("Spawing new async thread to treat {}", &media.id);
|
||||
let local_tweet_media_path = match get_tweet_media(&media, &cache_path).await {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
// we could have panicked here, no issue, but I’m not confortable using
|
||||
// that for now
|
||||
warn!("Cannot get tweet media for {}: {}", &media.url, e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// we cannot directly do all the stuff inside here because mastodon lock can
|
||||
// live outside this
|
||||
let mas_result = mastodon.lock().unwrap().media(Cow::from(String::from(&local_tweet_media_path)));
|
||||
|
||||
match mas_result {
|
||||
Ok(m) => {
|
||||
remove_file(&local_tweet_media_path).await.unwrap_or_else(|e|
|
||||
warn!("Attachment {} has been uploaded but I’m unable to remove the existing file: {}", &local_tweet_media_path, e)
|
||||
);
|
||||
// we can unwrap here because we’re in a thread
|
||||
tx.send(ScootalooSpawnResponse {
|
||||
mastodon_media_id: m.id.clone(),
|
||||
twitter_media_url: local_tweet_media_path.clone()
|
||||
}).await.unwrap();
|
||||
},
|
||||
Err(e) => {
|
||||
error!("Attachment {} cannot be uploaded to Mastodon Instance: {}", &local_tweet_media_path, e);
|
||||
}
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
status_medias.push(mastodon_media_ids);
|
||||
// dropping the last tx otherwise recv() will wait indefinitely
|
||||
drop(tx);
|
||||
|
||||
// last step, removing the reference to the media from with the toot’s text
|
||||
status_text = status_text.replace(&media.url, "");
|
||||
while let Some(i) = rx.recv().await {
|
||||
// pushes the media into the media vec
|
||||
status_medias.push(i.mastodon_media_id);
|
||||
// removes the URL from the original Tweet text
|
||||
status_text = status_text.replace(&i.twitter_media_url, "");
|
||||
}
|
||||
}
|
||||
// finished reuploading attachments, now let’s do the toot baby!
|
||||
|
||||
debug!("Building corresponding Mastodon status");
|
||||
let status = StatusBuilder::new()
|
||||
.status(&status_text)
|
||||
.media_ids(status_medias)
|
||||
@@ -397,7 +444,8 @@ pub fn run(config: Config) {
|
||||
.expect(format!("Cannot build status with text {}", &status_text).as_str());
|
||||
|
||||
// publish status
|
||||
mastodon.new_status(status).unwrap();
|
||||
// again unwrap is safe here as we are in the main thread
|
||||
mastodon.lock().unwrap().new_status(status).unwrap();
|
||||
// this will panic if it cannot publish the status, which is a good thing, it allows the
|
||||
// last_tweet gathered not to be written
|
||||
|
||||
|
||||
25
src/main.rs
25
src/main.rs
@@ -4,6 +4,13 @@ use scootaloo::*;
|
||||
// clap
|
||||
use clap::{App, Arg, SubCommand};
|
||||
|
||||
// log
|
||||
use log::{LevelFilter, error};
|
||||
use simple_logger::SimpleLogger;
|
||||
|
||||
// std
|
||||
use std::str::FromStr;
|
||||
|
||||
fn main() {
|
||||
let matches = App::new(env!("CARGO_PKG_NAME"))
|
||||
.version(env!("CARGO_PKG_VERSION"))
|
||||
@@ -15,6 +22,13 @@ fn main() {
|
||||
.help("TOML config file for scootaloo (default /usr/local/etc/scootaloo.toml)")
|
||||
.takes_value(true)
|
||||
.display_order(1))
|
||||
.arg(Arg::with_name("log_level")
|
||||
.short("l")
|
||||
.long("loglevel")
|
||||
.value_name("LOGLEVEL")
|
||||
.help("Log level. Valid values are: Off, Warn, Error, Info, Debug")
|
||||
.takes_value(true)
|
||||
.display_order(2))
|
||||
.subcommand(SubCommand::with_name("register")
|
||||
.version(env!("CARGO_PKG_VERSION"))
|
||||
.about("Command to register to a Mastodon Instance")
|
||||
@@ -32,7 +46,18 @@ fn main() {
|
||||
return;
|
||||
}
|
||||
|
||||
if matches.is_present("log_level") {
|
||||
match LevelFilter::from_str(matches.value_of("log_level").unwrap()) {
|
||||
Ok(level) => { SimpleLogger::new().with_level(level).init().unwrap()},
|
||||
Err(e) => {
|
||||
SimpleLogger::new().with_level(LevelFilter::Error).init().unwrap();
|
||||
error!("Unknown log level filter: {}", e);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let config = parse_toml(matches.value_of("config").unwrap_or("/usr/local/etc/scootaloo.toml"));
|
||||
|
||||
run(config);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user