diff --git a/.gitignore b/.gitignore index 72026a3..73b601b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /target -/templates \ No newline at end of file +/templates +/.vscode/* +!/.vscode/settings.json \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..2050082 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "rust-analyzer.cargo.features": [ + "btracker-crawler/i2p" + ] +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index f445044..6d27553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,52 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "amplify" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f7fb4ac7c881e54a8e7015e399b6112a2a5bc958b6c89ac510840ff20273b31" +dependencies = [ + "amplify_derive", + "amplify_num", + "ascii", + "getrandom 0.2.17", + "getrandom 0.3.4", + "wasm-bindgen", +] + +[[package]] +name = "amplify_derive" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a6309e6b8d89b36b9f959b7a8fa093583b94922a0f6438a24fb08936de4d428" +dependencies = [ + "amplify_syn", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "amplify_num" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afed304556696656d2d71495e1e5f2c4b524a3fb6eb0f2f3778ffc482a40b8a8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "amplify_syn" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7736fb8d473c0d83098b5bac44df6a561e20470375cd8bcae30516dc889fd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -103,6 +149,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "assert_cfg" version = "0.1.0" @@ -143,7 +195,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -154,7 +206,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -299,6 +351,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base32" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ce669cd6c8588f79e15cf450314f9638f967fc5770ff1c7c1deb0925ea7cfa" + [[package]] name = "base64" version = "0.22.1" @@ -394,15 +452,35 @@ dependencies = [ "serde", ] +[[package]] +name = "btpeer" +version = "0.8.0" +source = "git+https://codeberg.org/YGGverse/btpeer.git#0a2777b0f3cc9f4b37769f42ad0943fc7e1fefde" +dependencies = [ + "anyhow", + "bendy", + "clap", + "colored", + "cyphernet", + "data-encoding", + "rand 0.10.1", + "reqwest 0.13.4", + "serde", + "tokio", + "url", +] + [[package]] name = "btracker-crawler" version = "0.1.0" dependencies = [ "anyhow", "bendy", + "btpeer", "btracker-fs 0.3.1", "chrono", "clap", + "cyphernet", "librqbit", "log", "regex", @@ -411,6 +489,7 @@ dependencies = [ "tracing-subscriber", "url", "urlencoding", + "yosemite", ] [[package]] @@ -445,7 +524,7 @@ dependencies = [ [[package]] name = "btracker-gemini" -version = "0.1.3" +version = "0.2.0" dependencies = [ "anyhow", "btracker-fs 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -634,7 +713,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -658,6 +737,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "combine" version = "4.6.7" @@ -832,6 +920,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "cypheraddr" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4204e8808fcdd40bed39e49371f13f56d6984c32bc03dcb577d2a40b989b9d68" +dependencies = [ + "amplify", + "base32", + "cyphergraphy", +] + +[[package]] +name = "cyphergraphy" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23b157ed1e424149e1af82c6ed8300d41941756f5e9841396fd987d946ab0fe" +dependencies = [ + "amplify", + "ec25519", +] + +[[package]] +name = "cyphernet" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30958f36cb17f8d799e77d96e38e7686aabdc6308380a20268a736440325a7b0" +dependencies = [ + "cypheraddr", + "cyphergraphy", + "socks5-client", +] + [[package]] name = "darling" version = "0.23.0" @@ -852,7 +972,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -863,7 +983,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -933,7 +1053,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -975,7 +1095,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1002,6 +1122,25 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ec25519" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfd533a2fc01178c738c99412ae1f7e1ad2cb37c2e14bfd87e9d4618171c825" +dependencies = [ + "ed25519", + "getrandom 0.2.17", +] + +[[package]] +name = "ed25519" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +dependencies = [ + "signature", +] + [[package]] name = "either" version = "1.16.0" @@ -1198,7 +1337,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1930,7 +2069,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -1949,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2517,6 +2656,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2610,6 +2755,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -2766,7 +2921,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2845,7 +3000,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2884,7 +3039,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2999,7 +3154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -3019,7 +3174,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "version_check", "yansi", ] @@ -3263,7 +3418,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3455,7 +3610,7 @@ dependencies = [ "proc-macro2", "quote", "rocket_http", - "syn", + "syn 2.0.117", "unicode-xid", "version_check", ] @@ -3745,7 +3900,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3835,7 +3990,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3874,6 +4029,12 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" + [[package]] name = "simd-adler32" version = "0.3.9" @@ -3954,6 +4115,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks5-client" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a76ee33b601147d2d4b117252ebc70005d7938b7ad0e48a521243909702a115" +dependencies = [ + "amplify", + "cypheraddr", +] + [[package]] name = "spin" version = "0.9.8" @@ -4005,6 +4176,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -4033,7 +4215,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4124,7 +4306,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4135,7 +4317,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4240,7 +4422,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4392,6 +4574,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4405,7 +4588,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4677,7 +4860,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4859,7 +5042,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4870,7 +5053,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5194,7 +5377,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -5210,7 +5393,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5295,10 +5478,24 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] +[[package]] +name = "yosemite" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6bf3692263d7a9258016f5468c5cf5301b06189d7bc4c97b014b69022659871" +dependencies = [ + "futures", + "nom", + "rand 0.8.6", + "thiserror 1.0.69", + "tokio", + "tracing", +] + [[package]] name = "zerocopy" version = "0.8.50" @@ -5316,7 +5513,7 @@ checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5336,7 +5533,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -5376,7 +5573,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8225b7b..c68ec97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,4 +21,7 @@ default-members = [ #btracker-fs = { path = "crates/fs" } #librqbit = { git = "https://github.com/ikatson/rqbit.git", rev="ba07a75c9ad3bbb5e1d8aa8dee815ce8820fe372", package = "librqbit" } -#librqbit = { version = "9.0.0-beta.1", path = "../../rqbit/crates/librqbit", package = "librqbit" } \ No newline at end of file +#librqbit = { version = "9.0.0-beta.1", path = "../../rqbit/crates/librqbit", package = "librqbit" } + +btpeer = { git = "https://codeberg.org/YGGverse/btpeer.git" } +#btpeer = { path = "../../codeberg/yggverse/btpeer" } \ No newline at end of file diff --git a/crates/crawler/Cargo.toml b/crates/crawler/Cargo.toml index bd5a574..d2b61bc 100644 --- a/crates/crawler/Cargo.toml +++ b/crates/crawler/Cargo.toml @@ -10,12 +10,17 @@ categories = ["network-programming"] repository = "https://github.com/YGGverse/btracker" # homepage = "https://yggverse.github.io" +[features] +i2p = ["dep:btpeer", "dep:cyphernet", "dep:yosemite"] + [dependencies] anyhow = "1.0.102" bendy = "0.6.1" +btpeer = { version = "0.8.0", optional = true } btracker-fs = { version = "0.3.1", features = ["crawler"] } chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive"] } +cyphernet = { version = "0.5.4", features = ["i2p"], optional = true } librqbit = { version = "9.0.0-rc.0", features = ["disable-upload"]} log = "0.4.30" regex = "1.12.3" @@ -24,3 +29,4 @@ tokio = { version = "1.52.3", features = ["full"] } tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } url = "2.5.8" urlencoding = "2.1.3" +yosemite = { version = "0.7.0", optional = true } \ No newline at end of file diff --git a/crates/crawler/README.md b/crates/crawler/README.md index cd0ac31..b9dc905 100644 --- a/crates/crawler/README.md +++ b/crates/crawler/README.md @@ -6,4 +6,5 @@ SSD-friendly FS crawler of BEP 48 / Full Scrape, based on the [librqbit](https://github.com/ikatson/rqbit/tree/main/crates/librqbit) resolver > [!NOTE] -> By using OpenTracker as the scrape source, please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! \ No newline at end of file +> * By using OpenTracker as the index source, please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! +> * Enable experimental I2P support by `cargo build --features i2p` \ No newline at end of file diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index e367734..d500a4e 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -16,6 +16,16 @@ pub struct Config { #[arg(long, short)] pub full_scrape: Vec, + /// How long to wait for I2P tracker full scrape response + /// * by compiling OpenTracker for `/scrape`, + /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! + #[arg(long, default_value_t = 15)] + pub full_scrape_timeout: u64, + + /// Compact long data + #[arg(long, default_value = "gzip")] + pub full_scrape_compression: String, + /// The P2P Blocklist file URL (to filter outgoing connections) /// /// * use `--blocklist=file:///path/to/blocklist.txt` format for the local path @@ -94,6 +104,41 @@ pub struct Config { /// when the specified value in seconds is reached /// /// * the ban time is dynamically calculated based on the current ban list collected + /// * tip: increase this value when using I2P features #[arg(long, default_value_t = 60)] pub timeout: u64, + + /// Special trackers such as [opentracker-i2p](https://github.com/r4sas/opentracker-i2p) + /// that return B32 addresses instead of IP + #[cfg(feature = "i2p")] + #[arg(long)] + pub i2p_tracker: Vec, + + /// How long to wait for I2P tracker announce response + #[cfg(feature = "i2p")] + #[arg(long, default_value_t = 10)] + pub i2p_tracker_announce_timeout: u64, + + /// Absolute path(s) or URL(s) to the BEP 48 / Full Scrape for i2p_tracker(s) + /// * by compiling OpenTracker for `/scrape`, + /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! + #[cfg(feature = "i2p")] + #[arg(long, short)] + pub i2p_full_scrape: Vec, + + /// How long to wait for I2P tracker full scrape response + #[cfg(feature = "i2p")] + #[arg(long, default_value_t = 10)] + pub i2p_full_scrape_timeout: u64, + + /// Compact long data + #[cfg(feature = "i2p")] + #[arg(long, default_value = "gzip")] + pub i2p_full_scrape_compression: String, + + /// Use HTTP(s) proxy to resolve `i2p_tracker` and `i2p_full_scrape`, usually `http://127.0.0.1:4444` + /// * skip this setting if the I2P tracker is running locally (for the performance reasons) + #[cfg(feature = "i2p")] + #[arg(long)] + pub i2p_proxy: Option, } diff --git a/crates/crawler/src/full_scrape.rs b/crates/crawler/src/full_scrape.rs index feadaca..c4a982a 100644 --- a/crates/crawler/src/full_scrape.rs +++ b/crates/crawler/src/full_scrape.rs @@ -1,17 +1,64 @@ -use anyhow::Result; +use anyhow::{Result, bail}; use bendy::decoding::Decoder; use librqbit::dht::Id20; -use std::io; +use log::*; +use reqwest::{Client, Proxy, header::*}; +use std::{io, time::Duration}; /// Try parse info-hash from the given source, /// convert bytes to valid `InfoHash` v1 array on success. -pub async fn get(source: &str, capacity: usize) -> Result> { +pub async fn get( + source_url: &str, + capacity: usize, + timeout: Duration, + compression_method: &str, + http_proxy_url: Option<&str>, +) -> Result> { let mut i = Vec::with_capacity(capacity); - let bytes = if source.starts_with("http://") { - reqwest::get(source).await?.bytes().await?.into() + let bytes = if source_url.starts_with("http") { + trace!("build full scrape request to `{source_url}`..."); + let client = Client::builder().timeout(timeout); + let request = match http_proxy_url { + Some(p) => client.proxy(if p.starts_with("https") { + trace!("applying HTTPs proxy `{p}` to `{source_url}`..."); + Proxy::https(p)? + } else { + trace!("applying HTTP proxy `{p}` to `{source_url}`..."); + Proxy::http(p)? + }), + None => { + trace!("sending direct request to `{source_url}`..."); + client + } + } + .build()? + .get(source_url); + + trace!("sending full scrape request to `{source_url}`..."); + let response = if compression_method.is_empty() { + trace!("disable compression for `{source_url}`..."); + request + } else { + trace!("set `{compression_method}` compression method for `{source_url}`..."); + request.header(ACCEPT_ENCODING, compression_method) + } + .send() + .await?; + + if !response.status().is_success() { + bail!( + "HTTP request to `{source_url}` failed: {}", + response.status() + ) + } + + trace!("HTTP response from `{source_url}` received successfully, reading the bytes..."); + let res_bytes = response.bytes().await?; + res_bytes.to_vec() } else { - tokio::fs::read(source.trim_start_matches("file://")).await? + trace!("begin full scrape request from `{source_url}`..."); + tokio::fs::read(source_url.trim_start_matches("file://")).await? }; let mut decoder = Decoder::new(&bytes); @@ -20,17 +67,22 @@ pub async fn get(source: &str, capacity: usize) -> Result> { .next_object()? .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "empty"))?; + trace!("dictionary parse begin..."); let mut outer_dict = root_object.try_into_dictionary()?; while let Some(pair) = outer_dict.next_pair().unwrap_or(None) { if pair.0 == b"files" { + trace!("the `files` index found..."); let mut files_dict = pair .1 .try_into_dictionary() .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; while let Some(file_pair) = files_dict.next_pair().unwrap_or(None) { - i.push(Id20::from_bytes(file_pair.0)?) + let id20 = Id20::from_bytes(file_pair.0)?; + trace!("push `{}` to queue...", id20.as_string()); + i.push(id20) } + break; } } diff --git a/crates/crawler/src/i2p.rs b/crates/crawler/src/i2p.rs new file mode 100644 index 0000000..c5443a9 --- /dev/null +++ b/crates/crawler/src/i2p.rs @@ -0,0 +1,77 @@ +use anyhow::Result; +use log::*; +use std::{collections::HashSet, net::SocketAddr}; +use url::Url; +use yosemite::{Session, style::Stream}; + +/// Create virtual `SocketAddr` interfaces for given I2P trackers +pub async fn get_peers( + id20: &[u8; 20], + trackers: &Vec, + announce_timeout: u64, + trackers_proxy: Option<&Url>, +) -> Result> { + use btpeer::http::{announce, query::Announce}; + let mut peers = HashSet::new(); + for tracker in trackers { + debug!("get peers from I2P tracker `{tracker}`..."); + let mut peers_buffer = btpeer::peer::new_buffer(None); + let a = Announce::new(tracker.as_str(), id20, 0)?; + trace!("sending announce `{a}`..."); + announce( + &a, + std::time::Duration::from_secs(announce_timeout), + trackers_proxy.map(|url| url.as_str()), + Some(&mut peers_buffer), + ) + .await?; + let t = peers_buffer.len(); + trace!("received {t} peers total..."); + for (i, p) in peers_buffer.into_iter().enumerate() { + trace!("handle peer `{p}` ({i}/{t})..."); + match p.host { + cyphernet::addr::HostName::I2p(i2p) => match new_bridge(i2p.to_string()).await { + Ok(peer) => { + let s = peer.to_string(); + if peers.insert(peer) { + trace!("inserting new I2P peer `{s}`...") + } else { + trace!("replacing existing I2P peer `{s}`...") + } + } + Err(e) => warn!("could not create now socket for peer `{i2p}`: `{e}`; skip."), + }, + n => warn!("unexpected I2P address family: `{n}`; skip."), + } + } + } + debug!("collected {} unique peers total.", peers.len()); + trace!("virtual peer bridges returned: {peers:?}"); + Ok(peers) +} + +/// Create new outbound virtual stream to destination, +/// return local`SocketAddr` on success. +/// +/// Destination can be: +/// +/// * hostname such as `host.i2p` +/// * base32-encoded session received such as `lhbd7ojcaiofbfku7ixh47qj537g572zmhdc4oilvugzxdpdghua.b32.i2p/` +/// * base64-encoded string received from, e.g., `yosemite::Session::new` +async fn new_bridge(destination: String) -> Result { + let loopback = std::net::SocketAddrV4::new(std::net::Ipv4Addr::new(127, 0, 0, 1), 0); + + let listener = tokio::net::TcpListener::bind(loopback).await?; + let proxy_address = listener.local_addr()?; + let mut session = Session::::new(Default::default()).await?; + + tokio::spawn(async move { + while let Ok((mut local, _)) = listener.accept().await { + if let Ok(mut remote) = session.connect(&destination).await { + let _ = tokio::io::copy_bidirectional(&mut local, &mut remote).await; + } + } + }); + + Ok(proxy_address) +} diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index 5561503..c55fc2a 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -1,6 +1,9 @@ mod config; mod full_scrape; +#[cfg(feature = "i2p")] +mod i2p; + use anyhow::Result; use librqbit::{ AddTorrent, AddTorrentOptions, AddTorrentResponse, ConnectionOptions, DhtSessionConfig, @@ -90,19 +93,53 @@ async fn main() -> Result<()> { let mut queue = HashSet::with_capacity(config.index_capacity); for source in &config.full_scrape { debug!("index source `{source}`..."); - for i in match full_scrape::get(source, config.index_capacity).await { + for i in match full_scrape::get( + source, + config.index_capacity, + Duration::from_secs(config.full_scrape_timeout), + &config.full_scrape_compression, + None, + ) + .await + { Ok(i) => { debug!("fetch `{}` hashes from `{source}`...", i.len()); i } Err(e) => { - warn!("the feed `{source}` update failed: `{e}`; skip."); + warn!("the full scrape `{source}` update failed: `{e}`; skip."); continue; // skip without panic } } { queue.insert(i); } } + #[cfg(feature = "i2p")] + { + for source in &config.i2p_full_scrape { + debug!("index I2P source `{source}`..."); + for i in match full_scrape::get( + source, + config.index_capacity, + Duration::from_secs(config.i2p_full_scrape_timeout), + &config.i2p_full_scrape_compression, + config.i2p_proxy.as_ref().map(|p| p.as_str()), + ) + .await + { + Ok(i) => { + debug!("fetch `{}` hashes from I2P `{source}`...", i.len()); + i + } + Err(e) => { + warn!("I2P full scrape `{source}` update failed: `{e}`; skip."); + continue; // skip without panic + } + } { + queue.insert(i); + } + } + } // clean up nonexistent ban entries from the memory pool ban.retain(|i| { @@ -135,6 +172,8 @@ async fn main() -> Result<()> { debug!("torrent `{h}` is banned, skip for this queue."); continue; } + // init unique peers hash table + info!("resolve `{h}`..."); // run the crawler in single thread for performance reasons, // use `timeout` argument option to skip the dead connections. @@ -153,7 +192,37 @@ async fn main() -> Result<()> { paused: true, // continue after `only_files` update overwrite: true, disable_trackers: config.tracker.is_empty(), - initial_peers: config.initial_peer.clone(), + initial_peers: { + #[cfg(feature = "i2p")] + { + let mut peers: HashSet = config + .initial_peer + .as_ref() + .map(|p| p.iter().cloned().collect()) + .unwrap_or_default(); + match i2p::get_peers( + &i.0, + &config.i2p_tracker, + config.i2p_tracker_announce_timeout, + config.i2p_proxy.as_ref(), + ) + .await + { + Ok(p) => peers.extend(p), + Err(e) => warn!("{e}"), + } + if peers.is_empty() { + None + } else { + trace!("Collected {} unique peers to handle", peers.len()); + Some(peers.into_iter().collect()) + } + } + #[cfg(not(feature = "i2p"))] + { + config.initial_peer.clone() + } + }, list_only: false, // the destination folder to preload files match `preload_regex` // * e.g. images for audio albums @@ -247,9 +316,7 @@ async fn main() -> Result<()> { } } } - session.stop().await; - info!( "queue completed at {time_queue} (time: {} / uptime: {} / banned: {}) await {} seconds to continue...", Local::now() diff --git a/crates/gemini/Cargo.toml b/crates/gemini/Cargo.toml index 9a084a0..bbdb501 100644 --- a/crates/gemini/Cargo.toml +++ b/crates/gemini/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "btracker-gemini" -version = "0.1.3" +version = "0.2.0" edition = "2024" license = "MIT" readme = "README.md" diff --git a/crates/gemini/src/config.rs b/crates/gemini/src/config.rs index d701e03..b841e57 100644 --- a/crates/gemini/src/config.rs +++ b/crates/gemini/src/config.rs @@ -20,10 +20,14 @@ pub struct Config { #[arg(short, long, default_value_t = String::from("%Y/%m/%d"))] pub format_date: String, - /// Tracker(s) to join / scrape requests + /// Tracker(s) to public announce #[arg(short, long)] pub tracker: Option>, + /// Scrape(s) to local peers count resolve + #[arg(short, long)] + pub scrape: Option>, + /// Bind server `host:port` to listen incoming connections on it #[arg(short, long, default_value_t = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 1965)))] pub bind: SocketAddr, diff --git a/crates/gemini/src/main.rs b/crates/gemini/src/main.rs index be3141b..7068c3d 100644 --- a/crates/gemini/src/main.rs +++ b/crates/gemini/src/main.rs @@ -41,7 +41,7 @@ fn main() -> Result<()> { public: Storage::init(&config.storage, config.limit, config.capacity).unwrap(), scrape: Scrape::init( config - .tracker + .scrape .as_ref() .map(|u| { u.iter()