From 641bef65c556d62450415593393f2da7a816a133 Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Jun 2026 20:39:49 +0000 Subject: [PATCH 1/9] initial commit --- Cargo.lock | 263 ++++++++++++++++++++++++++++++----- Cargo.toml | 5 +- crates/crawler/Cargo.toml | 3 + crates/crawler/src/config.rs | 15 ++ crates/crawler/src/i2p.rs | 79 +++++++++++ crates/crawler/src/main.rs | 31 ++++- 6 files changed, 358 insertions(+), 38 deletions(-) create mode 100644 crates/crawler/src/i2p.rs diff --git a/Cargo.lock b/Cargo.lock index f445044..ab5c0fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,52 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "amplify" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f7fb4ac7c881e54a8e7015e399b6112a2a5bc958b6c89ac510840ff20273b31" +dependencies = [ + "amplify_derive", + "amplify_num", + "ascii", + "getrandom 0.2.17", + "getrandom 0.3.4", + "wasm-bindgen", +] + +[[package]] +name = "amplify_derive" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a6309e6b8d89b36b9f959b7a8fa093583b94922a0f6438a24fb08936de4d428" +dependencies = [ + "amplify_syn", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "amplify_num" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afed304556696656d2d71495e1e5f2c4b524a3fb6eb0f2f3778ffc482a40b8a8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "amplify_syn" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7736fb8d473c0d83098b5bac44df6a561e20470375cd8bcae30516dc889fd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -103,6 +149,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "assert_cfg" version = "0.1.0" @@ -143,7 +195,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -154,7 +206,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -299,6 +351,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base32" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ce669cd6c8588f79e15cf450314f9638f967fc5770ff1c7c1deb0925ea7cfa" + [[package]] name = "base64" version = "0.22.1" @@ -394,15 +452,35 @@ dependencies = [ "serde", ] +[[package]] +name = "btpeer" +version = "0.8.0" +source = "git+https://codeberg.org/YGGverse/btpeer.git#0a2777b0f3cc9f4b37769f42ad0943fc7e1fefde" +dependencies = [ + "anyhow", + "bendy", + "clap", + "colored", + "cyphernet", + "data-encoding", + "rand 0.10.1", + "reqwest 0.13.4", + "serde", + "tokio", + "url", +] + [[package]] name = "btracker-crawler" version = "0.1.0" dependencies = [ "anyhow", "bendy", + "btpeer", "btracker-fs 0.3.1", "chrono", "clap", + "cyphernet", "librqbit", "log", "regex", @@ -411,6 +489,7 @@ dependencies = [ "tracing-subscriber", "url", "urlencoding", + "yosemite", ] [[package]] @@ -634,7 +713,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -658,6 +737,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "combine" version = "4.6.7" @@ -832,6 +920,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "cypheraddr" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4204e8808fcdd40bed39e49371f13f56d6984c32bc03dcb577d2a40b989b9d68" +dependencies = [ + "amplify", + "base32", + "cyphergraphy", +] + +[[package]] +name = "cyphergraphy" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23b157ed1e424149e1af82c6ed8300d41941756f5e9841396fd987d946ab0fe" +dependencies = [ + "amplify", + "ec25519", +] + +[[package]] +name = "cyphernet" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30958f36cb17f8d799e77d96e38e7686aabdc6308380a20268a736440325a7b0" +dependencies = [ + "cypheraddr", + "cyphergraphy", + "socks5-client", +] + [[package]] name = "darling" version = "0.23.0" @@ -852,7 +972,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -863,7 +983,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -933,7 +1053,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -975,7 +1095,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1002,6 +1122,25 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ec25519" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfd533a2fc01178c738c99412ae1f7e1ad2cb37c2e14bfd87e9d4618171c825" +dependencies = [ + "ed25519", + "getrandom 0.2.17", +] + +[[package]] +name = "ed25519" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +dependencies = [ + "signature", +] + [[package]] name = "either" version = "1.16.0" @@ -1198,7 +1337,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1930,7 +2069,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -1949,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2517,6 +2656,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2610,6 +2755,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -2766,7 +2921,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2845,7 +3000,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2884,7 +3039,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2999,7 +3154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -3019,7 +3174,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "version_check", "yansi", ] @@ -3263,7 +3418,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3455,7 +3610,7 @@ dependencies = [ "proc-macro2", "quote", "rocket_http", - "syn", + "syn 2.0.117", "unicode-xid", "version_check", ] @@ -3745,7 +3900,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3835,7 +3990,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3874,6 +4029,12 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" + [[package]] name = "simd-adler32" version = "0.3.9" @@ -3954,6 +4115,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks5-client" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a76ee33b601147d2d4b117252ebc70005d7938b7ad0e48a521243909702a115" +dependencies = [ + "amplify", + "cypheraddr", +] + [[package]] name = "spin" version = "0.9.8" @@ -4005,6 +4176,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -4033,7 +4215,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4124,7 +4306,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4135,7 +4317,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4240,7 +4422,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4392,6 +4574,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4405,7 +4588,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4677,7 +4860,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4859,7 +5042,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4870,7 +5053,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5194,7 +5377,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -5210,7 +5393,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5295,10 +5478,24 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] +[[package]] +name = "yosemite" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6bf3692263d7a9258016f5468c5cf5301b06189d7bc4c97b014b69022659871" +dependencies = [ + "futures", + "nom", + "rand 0.8.6", + "thiserror 1.0.69", + "tokio", + "tracing", +] + [[package]] name = "zerocopy" version = "0.8.50" @@ -5316,7 +5513,7 @@ checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5336,7 +5533,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -5376,7 +5573,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8225b7b..c68ec97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,4 +21,7 @@ default-members = [ #btracker-fs = { path = "crates/fs" } #librqbit = { git = "https://github.com/ikatson/rqbit.git", rev="ba07a75c9ad3bbb5e1d8aa8dee815ce8820fe372", package = "librqbit" } -#librqbit = { version = "9.0.0-beta.1", path = "../../rqbit/crates/librqbit", package = "librqbit" } \ No newline at end of file +#librqbit = { version = "9.0.0-beta.1", path = "../../rqbit/crates/librqbit", package = "librqbit" } + +btpeer = { git = "https://codeberg.org/YGGverse/btpeer.git" } +#btpeer = { path = "../../codeberg/yggverse/btpeer" } \ No newline at end of file diff --git a/crates/crawler/Cargo.toml b/crates/crawler/Cargo.toml index bd5a574..9e9c875 100644 --- a/crates/crawler/Cargo.toml +++ b/crates/crawler/Cargo.toml @@ -13,9 +13,11 @@ repository = "https://github.com/YGGverse/btracker" [dependencies] anyhow = "1.0.102" bendy = "0.6.1" +btpeer = "0.8.0" btracker-fs = { version = "0.3.1", features = ["crawler"] } chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive"] } +cyphernet = { version = "0.5.4", features = ["i2p"] } librqbit = { version = "9.0.0-rc.0", features = ["disable-upload"]} log = "0.4.30" regex = "1.12.3" @@ -24,3 +26,4 @@ tokio = { version = "1.52.3", features = ["full"] } tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } url = "2.5.8" urlencoding = "2.1.3" +yosemite = "0.7.0" \ No newline at end of file diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index e367734..6436332 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -94,6 +94,21 @@ pub struct Config { /// when the specified value in seconds is reached /// /// * the ban time is dynamically calculated based on the current ban list collected + /// * tip: increase this value when using I2P features #[arg(long, default_value_t = 60)] pub timeout: u64, + + /// Special trackers such as [opentracker-i2p](https://github.com/r4sas/opentracker-i2p) + /// that return B32 addresses instead of IP + #[arg(long)] + pub i2p_tracker: Vec, + + /// Use HTTP(s) proxy to resolve `i2p_trackers`, usually `http://127.0.0.1:4444` + /// * skip this setting if the I2P tracker is running locally (for the performance reasons) + #[arg(long)] + pub i2p_tracker_proxy: Option, + + /// How long to wait for I2P tracker response + #[arg(long, default_value_t = 10)] + pub i2p_announce_timeout: u64, } diff --git a/crates/crawler/src/i2p.rs b/crates/crawler/src/i2p.rs new file mode 100644 index 0000000..36029ea --- /dev/null +++ b/crates/crawler/src/i2p.rs @@ -0,0 +1,79 @@ +use anyhow::Result; +use log::*; +use std::{collections::HashSet, net::SocketAddr}; +use url::Url; +use yosemite::{Session, style::Stream}; + +/// Create virtual `SocketAddr` interfaces for given I2P trackers +pub async fn get_peers( + id20: &[u8; 20], + trackers: &Vec, + announce_timeout: u64, + trackers_proxy: Option<&Url>, +) -> Result> { + use btpeer::http::{announce, query::Announce}; + let mut peers = HashSet::new(); + for tracker in trackers { + debug!("Get peers from I2P tracker `{tracker}`..."); + let mut peers_buffer = btpeer::peer::new_buffer(None); + let a = Announce::new(tracker.as_str(), id20, 0)?; + trace!("Sending announce `{a}`..."); + announce( + &a, + std::time::Duration::from_secs(announce_timeout), + trackers_proxy.map(|url| url.as_str()), + Some(&mut peers_buffer), + ) + .await?; + let t = peers_buffer.len(); + trace!("Received {t} peers total..."); + for (i, p) in peers_buffer.into_iter().enumerate() { + trace!("Handle peer `{p}` ({i}/{t})..."); + match p.host { + cyphernet::addr::HostName::I2p(i2p) => match new_bridge(i2p.to_string()).await { + Ok(peer) => { + let s = peer.to_string(); + if peers.insert(peer) { + trace!("Inserting new I2P peer `{s}`...") + } else { + trace!("Replacing existing I2P peer `{s}`...") + } + } + Err(e) => warn!("Could not create now socket for peer `{i2p}`: `{e}`; skip."), + }, + n => warn!("Unexpected I2P address family: `{n}`; skip."), + } + } + } + debug!("Collected {} unique peers total.", peers.len()); + trace!("Virtual peer bridges returned: {peers:?}"); + Ok(peers) +} + +/// Create new outbound virtual stream to destination, +/// return local`SocketAddr` on success. +/// +/// Destination can be: +/// +/// * hostname such as `host.i2p` +/// * base32-encoded session received such as `lhbd7ojcaiofbfku7ixh47qj537g572zmhdc4oilvugzxdpdghua.b32.i2p/` +/// * base64-encoded string received from, e.g., `yosemite::Session::new` +async fn new_bridge(destination: String) -> Result { + let loopback = std::net::SocketAddrV4::new(std::net::Ipv4Addr::new(127, 0, 0, 1), 0); + + let listener = tokio::net::TcpListener::bind(loopback).await?; + let proxy_address = listener.local_addr()?; + let mut session = Session::::new(Default::default()).await?; + + tokio::spawn({ + async move { + while let Ok((mut local, _)) = listener.accept().await { + if let Ok(mut remote) = session.connect(&destination).await { + let _ = tokio::io::copy_bidirectional(&mut local, &mut remote).await; + } + } + } + }); + + Ok(proxy_address) +} diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index 5561503..bc47f8b 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -1,12 +1,13 @@ mod config; mod full_scrape; +mod i2p; use anyhow::Result; use librqbit::{ AddTorrent, AddTorrentOptions, AddTorrentResponse, ConnectionOptions, DhtSessionConfig, Session, SessionOptions, }; -use std::{collections::HashSet, num::NonZero, time::Duration}; +use std::{collections::HashSet, net::SocketAddr, num::NonZero, time::Duration}; use url::Url; #[tokio::main] @@ -135,6 +136,12 @@ async fn main() -> Result<()> { debug!("torrent `{h}` is banned, skip for this queue."); continue; } + // init unique peers hash table + let mut initial_peers: HashSet = config + .initial_peer + .as_ref() + .map(|p| p.iter().cloned().collect()) + .unwrap_or_default(); info!("resolve `{h}`..."); // run the crawler in single thread for performance reasons, // use `timeout` argument option to skip the dead connections. @@ -153,7 +160,25 @@ async fn main() -> Result<()> { paused: true, // continue after `only_files` update overwrite: true, disable_trackers: config.tracker.is_empty(), - initial_peers: config.initial_peer.clone(), + initial_peers: { + match i2p::get_peers( + &i.0, + &config.i2p_tracker, + config.i2p_announce_timeout, + config.i2p_tracker_proxy.as_ref(), + ) + .await + { + Ok(p) => initial_peers.extend(p), + Err(e) => warn!("{e}"), + } + if initial_peers.is_empty() { + None + } else { + trace!("Collected {} unique peers to handle", initial_peers.len()); + Some(initial_peers.into_iter().collect()) + } + }, list_only: false, // the destination folder to preload files match `preload_regex` // * e.g. images for audio albums @@ -247,9 +272,7 @@ async fn main() -> Result<()> { } } } - session.stop().await; - info!( "queue completed at {time_queue} (time: {} / uptime: {} / banned: {}) await {} seconds to continue...", Local::now() From d1cec94af3fa55e41491203253de68807ed85aa5 Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Jun 2026 21:13:37 +0000 Subject: [PATCH 2/9] remove extra scopes --- crates/crawler/src/i2p.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/crawler/src/i2p.rs b/crates/crawler/src/i2p.rs index 36029ea..c801b94 100644 --- a/crates/crawler/src/i2p.rs +++ b/crates/crawler/src/i2p.rs @@ -65,12 +65,10 @@ async fn new_bridge(destination: String) -> Result { let proxy_address = listener.local_addr()?; let mut session = Session::::new(Default::default()).await?; - tokio::spawn({ - async move { - while let Ok((mut local, _)) = listener.accept().await { - if let Ok(mut remote) = session.connect(&destination).await { - let _ = tokio::io::copy_bidirectional(&mut local, &mut remote).await; - } + tokio::spawn(async move { + while let Ok((mut local, _)) = listener.accept().await { + if let Ok(mut remote) = session.connect(&destination).await { + let _ = tokio::io::copy_bidirectional(&mut local, &mut remote).await; } } }); From 5c67df22af25f458d4fd7e3c1bab38775cc13fb5 Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Jun 2026 22:53:24 +0000 Subject: [PATCH 3/9] lowercase debug messages --- crates/crawler/src/i2p.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/crawler/src/i2p.rs b/crates/crawler/src/i2p.rs index c801b94..c5443a9 100644 --- a/crates/crawler/src/i2p.rs +++ b/crates/crawler/src/i2p.rs @@ -14,10 +14,10 @@ pub async fn get_peers( use btpeer::http::{announce, query::Announce}; let mut peers = HashSet::new(); for tracker in trackers { - debug!("Get peers from I2P tracker `{tracker}`..."); + debug!("get peers from I2P tracker `{tracker}`..."); let mut peers_buffer = btpeer::peer::new_buffer(None); let a = Announce::new(tracker.as_str(), id20, 0)?; - trace!("Sending announce `{a}`..."); + trace!("sending announce `{a}`..."); announce( &a, std::time::Duration::from_secs(announce_timeout), @@ -26,27 +26,27 @@ pub async fn get_peers( ) .await?; let t = peers_buffer.len(); - trace!("Received {t} peers total..."); + trace!("received {t} peers total..."); for (i, p) in peers_buffer.into_iter().enumerate() { - trace!("Handle peer `{p}` ({i}/{t})..."); + trace!("handle peer `{p}` ({i}/{t})..."); match p.host { cyphernet::addr::HostName::I2p(i2p) => match new_bridge(i2p.to_string()).await { Ok(peer) => { let s = peer.to_string(); if peers.insert(peer) { - trace!("Inserting new I2P peer `{s}`...") + trace!("inserting new I2P peer `{s}`...") } else { - trace!("Replacing existing I2P peer `{s}`...") + trace!("replacing existing I2P peer `{s}`...") } } - Err(e) => warn!("Could not create now socket for peer `{i2p}`: `{e}`; skip."), + Err(e) => warn!("could not create now socket for peer `{i2p}`: `{e}`; skip."), }, - n => warn!("Unexpected I2P address family: `{n}`; skip."), + n => warn!("unexpected I2P address family: `{n}`; skip."), } } } - debug!("Collected {} unique peers total.", peers.len()); - trace!("Virtual peer bridges returned: {peers:?}"); + debug!("collected {} unique peers total.", peers.len()); + trace!("virtual peer bridges returned: {peers:?}"); Ok(peers) } From 94e1ee27577f59430868329cbbaffe6e0539aa92 Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Jun 2026 23:01:21 +0000 Subject: [PATCH 4/9] skip extra iterations --- crates/crawler/src/full_scrape.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/crawler/src/full_scrape.rs b/crates/crawler/src/full_scrape.rs index feadaca..2708ea1 100644 --- a/crates/crawler/src/full_scrape.rs +++ b/crates/crawler/src/full_scrape.rs @@ -31,6 +31,7 @@ pub async fn get(source: &str, capacity: usize) -> Result> { while let Some(file_pair) = files_dict.next_pair().unwrap_or(None) { i.push(Id20::from_bytes(file_pair.0)?) } + break; } } From 6afd39574bbdee12da63d84163f40630d3ea7aeb Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Jun 2026 23:35:34 +0000 Subject: [PATCH 5/9] implement i2p full scrape request; improve debug output --- crates/crawler/src/config.rs | 28 +++++++++++++---- crates/crawler/src/full_scrape.rs | 52 ++++++++++++++++++++++++++----- crates/crawler/src/main.rs | 37 +++++++++++++++++++--- 3 files changed, 100 insertions(+), 17 deletions(-) diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index 6436332..d1d8132 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -16,6 +16,12 @@ pub struct Config { #[arg(long, short)] pub full_scrape: Vec, + /// How long to wait for I2P tracker full scrape response + /// * by compiling OpenTracker for `/scrape`, + /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! + #[arg(long, default_value_t = 15)] + pub full_scrape_timeout: u64, + /// The P2P Blocklist file URL (to filter outgoing connections) /// /// * use `--blocklist=file:///path/to/blocklist.txt` format for the local path @@ -103,12 +109,22 @@ pub struct Config { #[arg(long)] pub i2p_tracker: Vec, - /// Use HTTP(s) proxy to resolve `i2p_trackers`, usually `http://127.0.0.1:4444` - /// * skip this setting if the I2P tracker is running locally (for the performance reasons) - #[arg(long)] - pub i2p_tracker_proxy: Option, + /// How long to wait for I2P tracker announce response + #[arg(long, default_value_t = 10)] + pub i2p_tracker_announce_timeout: u64, - /// How long to wait for I2P tracker response + /// Absolute path(s) or URL(s) to the BEP 48 / Full Scrape for i2p_tracker(s) + /// * by compiling OpenTracker for `/scrape`, + /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! + #[arg(long, short)] + pub i2p_full_scrape: Vec, + + /// How long to wait for I2P tracker full scrape response #[arg(long, default_value_t = 10)] - pub i2p_announce_timeout: u64, + pub i2p_full_scrape_timeout: u64, + + /// Use HTTP(s) proxy to resolve `i2p_tracker` and `i2p_full_scrape`, usually `http://127.0.0.1:4444` + /// * skip this setting if the I2P tracker is running locally (for the performance reasons) + #[arg(long)] + pub i2p_proxy: Option, } diff --git a/crates/crawler/src/full_scrape.rs b/crates/crawler/src/full_scrape.rs index 2708ea1..79226a5 100644 --- a/crates/crawler/src/full_scrape.rs +++ b/crates/crawler/src/full_scrape.rs @@ -1,17 +1,51 @@ -use anyhow::Result; +use anyhow::{Result, bail}; use bendy::decoding::Decoder; use librqbit::dht::Id20; -use std::io; +use log::*; +use reqwest::{Client, Proxy}; +use std::{io, time::Duration}; /// Try parse info-hash from the given source, /// convert bytes to valid `InfoHash` v1 array on success. -pub async fn get(source: &str, capacity: usize) -> Result> { +pub async fn get( + source_url: &str, + capacity: usize, + timeout: Duration, + http_proxy_url: Option<&str>, +) -> Result> { let mut i = Vec::with_capacity(capacity); - let bytes = if source.starts_with("http://") { - reqwest::get(source).await?.bytes().await?.into() + let bytes = if source_url.starts_with("http") { + trace!("begin full scrape request to `{source_url}`..."); + let client = Client::builder().timeout(timeout); + let response = match http_proxy_url { + Some(p) => client.proxy(if p.starts_with("https") { + trace!("applying HTTPs proxy `{p}` to `{source_url}`..."); + Proxy::https(p)? + } else { + trace!("applying HTTP proxy `{p}` to `{source_url}`..."); + Proxy::http(p)? + }), + None => { + trace!("sending direct request to `{source_url}`..."); + client + } + } + .build()? + .get(source_url) + .send() + .await?; + + if !response.status().is_success() { + bail!("HTTP error: {}", response.status()) + } + + trace!("HTTP response received successfully, reading the bytes..."); + let res_bytes = response.bytes().await?; + res_bytes.to_vec() } else { - tokio::fs::read(source.trim_start_matches("file://")).await? + trace!("begin full scrape request from `{source_url}`..."); + tokio::fs::read(source_url.trim_start_matches("file://")).await? }; let mut decoder = Decoder::new(&bytes); @@ -20,16 +54,20 @@ pub async fn get(source: &str, capacity: usize) -> Result> { .next_object()? .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "empty"))?; + trace!("dictionary parse begin..."); let mut outer_dict = root_object.try_into_dictionary()?; while let Some(pair) = outer_dict.next_pair().unwrap_or(None) { if pair.0 == b"files" { + trace!("the `files` index found..."); let mut files_dict = pair .1 .try_into_dictionary() .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; while let Some(file_pair) = files_dict.next_pair().unwrap_or(None) { - i.push(Id20::from_bytes(file_pair.0)?) + let id20 = Id20::from_bytes(file_pair.0)?; + trace!("push `{}` to queue...", id20.as_string()); + i.push(id20) } break; } diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index bc47f8b..be91372 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -91,13 +91,42 @@ async fn main() -> Result<()> { let mut queue = HashSet::with_capacity(config.index_capacity); for source in &config.full_scrape { debug!("index source `{source}`..."); - for i in match full_scrape::get(source, config.index_capacity).await { + for i in match full_scrape::get( + source, + config.index_capacity, + Duration::from_secs(config.full_scrape_timeout), + None, + ) + .await + { Ok(i) => { debug!("fetch `{}` hashes from `{source}`...", i.len()); i } Err(e) => { - warn!("the feed `{source}` update failed: `{e}`; skip."); + warn!("the full scrape `{source}` update failed: `{e}`; skip."); + continue; // skip without panic + } + } { + queue.insert(i); + } + } + for source in &config.i2p_full_scrape { + debug!("index I2P source `{source}`..."); + for i in match full_scrape::get( + source, + config.index_capacity, + Duration::from_secs(config.i2p_full_scrape_timeout), + config.i2p_proxy.as_ref().map(|p| p.as_str()), + ) + .await + { + Ok(i) => { + debug!("fetch `{}` hashes from I2P `{source}`...", i.len()); + i + } + Err(e) => { + warn!("I2P full scrape `{source}` update failed: `{e}`; skip."); continue; // skip without panic } } { @@ -164,8 +193,8 @@ async fn main() -> Result<()> { match i2p::get_peers( &i.0, &config.i2p_tracker, - config.i2p_announce_timeout, - config.i2p_tracker_proxy.as_ref(), + config.i2p_tracker_announce_timeout, + config.i2p_proxy.as_ref(), ) .await { From a046da36540d77f117b18860029a3b0a303b948c Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Jun 2026 23:52:02 +0000 Subject: [PATCH 6/9] implement compression settings --- crates/crawler/src/config.rs | 8 ++++++++ crates/crawler/src/full_scrape.rs | 25 +++++++++++++++++++------ crates/crawler/src/main.rs | 2 ++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index d1d8132..94ff310 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -22,6 +22,10 @@ pub struct Config { #[arg(long, default_value_t = 15)] pub full_scrape_timeout: u64, + /// Compact long data + #[arg(long, default_value = "gzip")] + pub full_scrape_compression: String, + /// The P2P Blocklist file URL (to filter outgoing connections) /// /// * use `--blocklist=file:///path/to/blocklist.txt` format for the local path @@ -123,6 +127,10 @@ pub struct Config { #[arg(long, default_value_t = 10)] pub i2p_full_scrape_timeout: u64, + /// Compact long data + #[arg(long, default_value = "gzip")] + pub i2p_full_scrape_compression: String, + /// Use HTTP(s) proxy to resolve `i2p_tracker` and `i2p_full_scrape`, usually `http://127.0.0.1:4444` /// * skip this setting if the I2P tracker is running locally (for the performance reasons) #[arg(long)] diff --git a/crates/crawler/src/full_scrape.rs b/crates/crawler/src/full_scrape.rs index 79226a5..c4a982a 100644 --- a/crates/crawler/src/full_scrape.rs +++ b/crates/crawler/src/full_scrape.rs @@ -2,7 +2,7 @@ use anyhow::{Result, bail}; use bendy::decoding::Decoder; use librqbit::dht::Id20; use log::*; -use reqwest::{Client, Proxy}; +use reqwest::{Client, Proxy, header::*}; use std::{io, time::Duration}; /// Try parse info-hash from the given source, @@ -11,14 +11,15 @@ pub async fn get( source_url: &str, capacity: usize, timeout: Duration, + compression_method: &str, http_proxy_url: Option<&str>, ) -> Result> { let mut i = Vec::with_capacity(capacity); let bytes = if source_url.starts_with("http") { - trace!("begin full scrape request to `{source_url}`..."); + trace!("build full scrape request to `{source_url}`..."); let client = Client::builder().timeout(timeout); - let response = match http_proxy_url { + let request = match http_proxy_url { Some(p) => client.proxy(if p.starts_with("https") { trace!("applying HTTPs proxy `{p}` to `{source_url}`..."); Proxy::https(p)? @@ -32,15 +33,27 @@ pub async fn get( } } .build()? - .get(source_url) + .get(source_url); + + trace!("sending full scrape request to `{source_url}`..."); + let response = if compression_method.is_empty() { + trace!("disable compression for `{source_url}`..."); + request + } else { + trace!("set `{compression_method}` compression method for `{source_url}`..."); + request.header(ACCEPT_ENCODING, compression_method) + } .send() .await?; if !response.status().is_success() { - bail!("HTTP error: {}", response.status()) + bail!( + "HTTP request to `{source_url}` failed: {}", + response.status() + ) } - trace!("HTTP response received successfully, reading the bytes..."); + trace!("HTTP response from `{source_url}` received successfully, reading the bytes..."); let res_bytes = response.bytes().await?; res_bytes.to_vec() } else { diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index be91372..e03f949 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -95,6 +95,7 @@ async fn main() -> Result<()> { source, config.index_capacity, Duration::from_secs(config.full_scrape_timeout), + &config.full_scrape_compression, None, ) .await @@ -117,6 +118,7 @@ async fn main() -> Result<()> { source, config.index_capacity, Duration::from_secs(config.i2p_full_scrape_timeout), + &config.i2p_full_scrape_compression, config.i2p_proxy.as_ref().map(|p| p.as_str()), ) .await From 7fcdd553819a1d0dbbe47e721af5411733f50a1d Mon Sep 17 00:00:00 2001 From: yggverse Date: Sun, 7 Jun 2026 00:21:49 +0000 Subject: [PATCH 7/9] set i2p as feature --- .gitignore | 4 +- .vscode/settings.json | 5 ++ crates/crawler/Cargo.toml | 9 ++-- crates/crawler/README.md | 3 +- crates/crawler/src/config.rs | 6 +++ crates/crawler/src/main.rs | 93 ++++++++++++++++++++---------------- 6 files changed, 75 insertions(+), 45 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 72026a3..73b601b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /target -/templates \ No newline at end of file +/templates +/.vscode/* +!/.vscode/settings.json \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..8bf7ea6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "rust-analyzer.cargo.features": [ + "i2p" + ] +} \ No newline at end of file diff --git a/crates/crawler/Cargo.toml b/crates/crawler/Cargo.toml index 9e9c875..d2b61bc 100644 --- a/crates/crawler/Cargo.toml +++ b/crates/crawler/Cargo.toml @@ -10,14 +10,17 @@ categories = ["network-programming"] repository = "https://github.com/YGGverse/btracker" # homepage = "https://yggverse.github.io" +[features] +i2p = ["dep:btpeer", "dep:cyphernet", "dep:yosemite"] + [dependencies] anyhow = "1.0.102" bendy = "0.6.1" -btpeer = "0.8.0" +btpeer = { version = "0.8.0", optional = true } btracker-fs = { version = "0.3.1", features = ["crawler"] } chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive"] } -cyphernet = { version = "0.5.4", features = ["i2p"] } +cyphernet = { version = "0.5.4", features = ["i2p"], optional = true } librqbit = { version = "9.0.0-rc.0", features = ["disable-upload"]} log = "0.4.30" regex = "1.12.3" @@ -26,4 +29,4 @@ tokio = { version = "1.52.3", features = ["full"] } tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } url = "2.5.8" urlencoding = "2.1.3" -yosemite = "0.7.0" \ No newline at end of file +yosemite = { version = "0.7.0", optional = true } \ No newline at end of file diff --git a/crates/crawler/README.md b/crates/crawler/README.md index cd0ac31..b9dc905 100644 --- a/crates/crawler/README.md +++ b/crates/crawler/README.md @@ -6,4 +6,5 @@ SSD-friendly FS crawler of BEP 48 / Full Scrape, based on the [librqbit](https://github.com/ikatson/rqbit/tree/main/crates/librqbit) resolver > [!NOTE] -> By using OpenTracker as the scrape source, please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! \ No newline at end of file +> * By using OpenTracker as the index source, please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! +> * Enable experimental I2P support by `cargo build --features i2p` \ No newline at end of file diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index 94ff310..d500a4e 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -110,29 +110,35 @@ pub struct Config { /// Special trackers such as [opentracker-i2p](https://github.com/r4sas/opentracker-i2p) /// that return B32 addresses instead of IP + #[cfg(feature = "i2p")] #[arg(long)] pub i2p_tracker: Vec, /// How long to wait for I2P tracker announce response + #[cfg(feature = "i2p")] #[arg(long, default_value_t = 10)] pub i2p_tracker_announce_timeout: u64, /// Absolute path(s) or URL(s) to the BEP 48 / Full Scrape for i2p_tracker(s) /// * by compiling OpenTracker for `/scrape`, /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! + #[cfg(feature = "i2p")] #[arg(long, short)] pub i2p_full_scrape: Vec, /// How long to wait for I2P tracker full scrape response + #[cfg(feature = "i2p")] #[arg(long, default_value_t = 10)] pub i2p_full_scrape_timeout: u64, /// Compact long data + #[cfg(feature = "i2p")] #[arg(long, default_value = "gzip")] pub i2p_full_scrape_compression: String, /// Use HTTP(s) proxy to resolve `i2p_tracker` and `i2p_full_scrape`, usually `http://127.0.0.1:4444` /// * skip this setting if the I2P tracker is running locally (for the performance reasons) + #[cfg(feature = "i2p")] #[arg(long)] pub i2p_proxy: Option, } diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index e03f949..c55fc2a 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -1,5 +1,7 @@ mod config; mod full_scrape; + +#[cfg(feature = "i2p")] mod i2p; use anyhow::Result; @@ -7,7 +9,7 @@ use librqbit::{ AddTorrent, AddTorrentOptions, AddTorrentResponse, ConnectionOptions, DhtSessionConfig, Session, SessionOptions, }; -use std::{collections::HashSet, net::SocketAddr, num::NonZero, time::Duration}; +use std::{collections::HashSet, num::NonZero, time::Duration}; use url::Url; #[tokio::main] @@ -112,27 +114,30 @@ async fn main() -> Result<()> { queue.insert(i); } } - for source in &config.i2p_full_scrape { - debug!("index I2P source `{source}`..."); - for i in match full_scrape::get( - source, - config.index_capacity, - Duration::from_secs(config.i2p_full_scrape_timeout), - &config.i2p_full_scrape_compression, - config.i2p_proxy.as_ref().map(|p| p.as_str()), - ) - .await - { - Ok(i) => { - debug!("fetch `{}` hashes from I2P `{source}`...", i.len()); - i - } - Err(e) => { - warn!("I2P full scrape `{source}` update failed: `{e}`; skip."); - continue; // skip without panic + #[cfg(feature = "i2p")] + { + for source in &config.i2p_full_scrape { + debug!("index I2P source `{source}`..."); + for i in match full_scrape::get( + source, + config.index_capacity, + Duration::from_secs(config.i2p_full_scrape_timeout), + &config.i2p_full_scrape_compression, + config.i2p_proxy.as_ref().map(|p| p.as_str()), + ) + .await + { + Ok(i) => { + debug!("fetch `{}` hashes from I2P `{source}`...", i.len()); + i + } + Err(e) => { + warn!("I2P full scrape `{source}` update failed: `{e}`; skip."); + continue; // skip without panic + } + } { + queue.insert(i); } - } { - queue.insert(i); } } @@ -168,11 +173,7 @@ async fn main() -> Result<()> { continue; } // init unique peers hash table - let mut initial_peers: HashSet = config - .initial_peer - .as_ref() - .map(|p| p.iter().cloned().collect()) - .unwrap_or_default(); + info!("resolve `{h}`..."); // run the crawler in single thread for performance reasons, // use `timeout` argument option to skip the dead connections. @@ -192,22 +193,34 @@ async fn main() -> Result<()> { overwrite: true, disable_trackers: config.tracker.is_empty(), initial_peers: { - match i2p::get_peers( - &i.0, - &config.i2p_tracker, - config.i2p_tracker_announce_timeout, - config.i2p_proxy.as_ref(), - ) - .await + #[cfg(feature = "i2p")] { - Ok(p) => initial_peers.extend(p), - Err(e) => warn!("{e}"), + let mut peers: HashSet = config + .initial_peer + .as_ref() + .map(|p| p.iter().cloned().collect()) + .unwrap_or_default(); + match i2p::get_peers( + &i.0, + &config.i2p_tracker, + config.i2p_tracker_announce_timeout, + config.i2p_proxy.as_ref(), + ) + .await + { + Ok(p) => peers.extend(p), + Err(e) => warn!("{e}"), + } + if peers.is_empty() { + None + } else { + trace!("Collected {} unique peers to handle", peers.len()); + Some(peers.into_iter().collect()) + } } - if initial_peers.is_empty() { - None - } else { - trace!("Collected {} unique peers to handle", initial_peers.len()); - Some(initial_peers.into_iter().collect()) + #[cfg(not(feature = "i2p"))] + { + config.initial_peer.clone() } }, list_only: false, From 6d8b3b893442eafc8394dce6c59647dd159f7e7c Mon Sep 17 00:00:00 2001 From: yggverse Date: Sun, 7 Jun 2026 11:46:35 +0000 Subject: [PATCH 8/9] fix feature namespace --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8bf7ea6..2050082 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { "rust-analyzer.cargo.features": [ - "i2p" + "btracker-crawler/i2p" ] } \ No newline at end of file From 18026bad552bd0ed6b2476435c82b74fc652422f Mon Sep 17 00:00:00 2001 From: yggverse Date: Sun, 7 Jun 2026 11:47:26 +0000 Subject: [PATCH 9/9] implement separated `scrape` config option; update version --- Cargo.lock | 2 +- crates/gemini/Cargo.toml | 2 +- crates/gemini/src/config.rs | 6 +++++- crates/gemini/src/main.rs | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab5c0fe..6d27553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -524,7 +524,7 @@ dependencies = [ [[package]] name = "btracker-gemini" -version = "0.1.3" +version = "0.2.0" dependencies = [ "anyhow", "btracker-fs 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/crates/gemini/Cargo.toml b/crates/gemini/Cargo.toml index 9a084a0..bbdb501 100644 --- a/crates/gemini/Cargo.toml +++ b/crates/gemini/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "btracker-gemini" -version = "0.1.3" +version = "0.2.0" edition = "2024" license = "MIT" readme = "README.md" diff --git a/crates/gemini/src/config.rs b/crates/gemini/src/config.rs index d701e03..b841e57 100644 --- a/crates/gemini/src/config.rs +++ b/crates/gemini/src/config.rs @@ -20,10 +20,14 @@ pub struct Config { #[arg(short, long, default_value_t = String::from("%Y/%m/%d"))] pub format_date: String, - /// Tracker(s) to join / scrape requests + /// Tracker(s) to public announce #[arg(short, long)] pub tracker: Option>, + /// Scrape(s) to local peers count resolve + #[arg(short, long)] + pub scrape: Option>, + /// Bind server `host:port` to listen incoming connections on it #[arg(short, long, default_value_t = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 1965)))] pub bind: SocketAddr, diff --git a/crates/gemini/src/main.rs b/crates/gemini/src/main.rs index be3141b..7068c3d 100644 --- a/crates/gemini/src/main.rs +++ b/crates/gemini/src/main.rs @@ -41,7 +41,7 @@ fn main() -> Result<()> { public: Storage::init(&config.storage, config.limit, config.capacity).unwrap(), scrape: Scrape::init( config - .tracker + .scrape .as_ref() .map(|u| { u.iter()