diff --git a/.gitignore b/.gitignore index 73b601b..72026a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ /target -/templates -/.vscode/* -!/.vscode/settings.json \ No newline at end of file +/templates \ No newline at end of file diff --git a/.vscode/.gitignore b/.vscode/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 2050082..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "rust-analyzer.cargo.features": [ - "btracker-crawler/i2p" - ] -} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 6d27553..7756729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arc-swap" @@ -145,9 +145,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" [[package]] name = "ascii" @@ -195,7 +195,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -206,7 +206,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -276,7 +276,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "hyper 1.10.1", @@ -304,7 +304,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "mime", @@ -326,7 +326,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "mime", @@ -407,15 +407,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -432,6 +432,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bs58" version = "0.5.1" @@ -443,19 +452,20 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] name = "btpeer" -version = "0.8.0" -source = "git+https://codeberg.org/YGGverse/btpeer.git#0a2777b0f3cc9f4b37769f42ad0943fc7e1fefde" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44be4fbde7eb7b8948851799471e78d340e65e9b5713e9d02aa7a18cbcd7d9f0" dependencies = [ "anyhow", "bendy", @@ -463,6 +473,7 @@ dependencies = [ "colored", "cyphernet", "data-encoding", + "dns-lookup", "rand 0.10.1", "reqwest 0.13.4", "serde", @@ -472,39 +483,32 @@ dependencies = [ [[package]] name = "btracker-crawler" -version = "0.1.0" +version = "0.3.0" dependencies = [ "anyhow", - "bendy", "btpeer", - "btracker-fs 0.3.1", + "btracker-fs 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", "clap", "cyphernet", + "data-encoding", "librqbit", "log", + "rand 0.10.1", "regex", - "reqwest 0.13.4", + "serde", + "serde-inline-default", + "sha2 0.11.0", "tokio", + "toml 1.1.2+spec-1.1.0", "tracing-subscriber", "url", - "urlencoding", "yosemite", ] [[package]] name = "btracker-fs" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6846910a99e54812b66ce31c1a42f2690fdfc0b8afcfe162196fd1a2af1dbf9a" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "btracker-fs" -version = "0.4.0" +version = "0.5.0" dependencies = [ "chrono", "librqbit-core 5.0.0", @@ -514,21 +518,23 @@ dependencies = [ [[package]] name = "btracker-fs" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b70704bdaaa34d4655f6765195ae3da950b5fc2dd2f14cf267defae13b8bea7" +checksum = "113b229b0535afe4589e2c3a0805761b9d592253faab3c55b6b69b79614e5e36" dependencies = [ "chrono", "librqbit-core 5.0.0", + "log", + "regex", ] [[package]] name = "btracker-gemini" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", - "btracker-fs 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "btracker-scrape 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "btracker-fs 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "btracker-scrape", "chrono", "clap", "librqbit-core 5.0.0", @@ -537,6 +543,7 @@ dependencies = [ "plurify", "regex", "titanite", + "tokio", "tracing-subscriber", "url", "urlencoding", @@ -544,10 +551,11 @@ dependencies = [ [[package]] name = "btracker-http" -version = "0.1.1" +version = "0.2.0" dependencies = [ - "btracker-fs 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "btracker-scrape 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "btpeer", + "btracker-fs 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "btracker-scrape", "chrono", "clap", "librqbit-core 5.0.0", @@ -562,18 +570,11 @@ dependencies = [ [[package]] name = "btracker-scrape" -version = "0.1.0" -dependencies = [ - "rand 0.9.4", -] - -[[package]] -name = "btracker-scrape" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f80f29bb8c0fdee1f2b8c83ee590339bab6e8db2cb9ee601546f1d95005e7" +version = "0.2.0" dependencies = [ - "rand 0.9.4", + "anyhow", + "btpeer", + "url", ] [[package]] @@ -596,15 +597,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "cc" -version = "1.2.63" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -626,9 +627,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -713,7 +714,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -791,6 +792,12 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const_panic" version = "0.2.15" @@ -908,6 +915,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "crypto-hash" version = "0.3.4" @@ -972,7 +988,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -983,7 +999,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1013,7 +1029,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -1049,11 +1064,11 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1062,8 +1077,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", ] [[package]] @@ -1095,7 +1121,19 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", +] + +[[package]] +name = "dns-lookup" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" +dependencies = [ + "cfg-if", + "libc", + "socket2 0.6.4", + "windows-sys 0.60.2", ] [[package]] @@ -1187,7 +1225,7 @@ dependencies = [ "atomic 0.6.1", "pear", "serde", - "toml", + "toml 0.8.23", "uncased", "version_check", ] @@ -1224,12 +1262,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foldhash" version = "0.2.0" @@ -1337,7 +1369,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1436,16 +1468,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", ] [[package]] @@ -1473,7 +1503,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "ignore", "walkdir", ] @@ -1534,16 +1564,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.1", + "http 1.4.2", "indexmap 2.14.0", "slab", "tokio", @@ -1563,15 +1593,6 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -1580,7 +1601,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1626,9 +1647,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1652,7 +1673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.1", + "http 1.4.2", ] [[package]] @@ -1663,7 +1684,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "pin-project-lite", ] @@ -1689,6 +1710,15 @@ dependencies = [ "libm", ] +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -1723,8 +1753,8 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.14", - "http 1.4.1", + "h2 0.4.15", + "http 1.4.2", "http-body 1.0.1", "httparse", "httpdate", @@ -1741,7 +1771,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.1", + "http 1.4.2", "hyper 1.10.1", "hyper-util", "rustls", @@ -1776,7 +1806,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "hyper 1.10.1", "ipnet", @@ -1897,12 +1927,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1932,9 +1956,9 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a" +checksum = "b915661dd01db3f05050265b2477bcc6527b3792388e2749b41623cc592be67d" dependencies = [ "crossbeam-deque", "globset", @@ -2069,7 +2093,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2088,7 +2112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2103,13 +2127,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2129,7 +2152,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "libc", ] @@ -2150,12 +2173,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" version = "0.2.186" @@ -2199,7 +2216,7 @@ dependencies = [ "futures", "governor", "hex 0.4.3", - "http 1.4.1", + "http 1.4.2", "intervaltree", "itertools", "librqbit-bencode 9.0.0-rc.0", @@ -2575,9 +2592,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "loom" @@ -2617,15 +2634,15 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -2704,7 +2721,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 1.4.1", + "http 1.4.2", "httparse", "memchr", "mime", @@ -2749,7 +2766,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libc", @@ -2786,7 +2803,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -2901,11 +2918,11 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl" -version = "0.10.80" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "cfg-if", "foreign-types", "libc", @@ -2921,7 +2938,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2932,9 +2949,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.116" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -3000,7 +3017,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3039,7 +3056,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3049,7 +3066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -3147,16 +3164,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -3174,7 +3181,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "version_check", "yansi", ] @@ -3206,9 +3213,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -3226,9 +3233,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", @@ -3262,9 +3269,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -3315,7 +3322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -3378,7 +3385,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", ] [[package]] @@ -3387,7 +3394,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", ] [[package]] @@ -3418,14 +3425,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -3446,9 +3453,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -3460,7 +3467,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "hyper 1.10.1", @@ -3499,8 +3506,8 @@ dependencies = [ "bytes", "encoding_rs", "futures-core", - "h2 0.4.14", - "http 1.4.1", + "h2 0.4.15", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "hyper 1.10.1", @@ -3610,7 +3617,7 @@ dependencies = [ "proc-macro2", "quote", "rocket_http", - "syn 2.0.117", + "syn 2.0.118", "unicode-xid", "version_check", ] @@ -3676,7 +3683,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys", @@ -3685,9 +3692,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "once_cell", @@ -3830,7 +3837,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3863,6 +3870,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-inline-default" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf03b7a5281cfd4013bc788d057b0f09fc634397d83bc8973c955bd4f32727a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "serde_bencode" version = "0.2.4" @@ -3900,7 +3918,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3949,6 +3967,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3963,9 +3990,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64", "bs58", @@ -3983,14 +4010,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4001,7 +4028,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4091,9 +4129,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" @@ -4189,9 +4227,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -4215,7 +4253,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4224,7 +4262,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4252,7 +4290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4306,7 +4344,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4317,7 +4355,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4331,12 +4369,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -4346,15 +4383,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -4422,7 +4459,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4489,11 +4526,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_edit", ] +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap 2.14.0", + "serde_core", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -4503,6 +4555,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -4511,10 +4572,19 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.14.0", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.3", ] [[package]] @@ -4523,6 +4593,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "tower" version = "0.5.3" @@ -4544,10 +4620,10 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.12.1", + "bitflags 2.13.0", "bytes", "futures-util", - "http 1.4.1", + "http 1.4.2", "http-body 1.0.1", "pin-project-lite", "tower", @@ -4588,7 +4664,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4742,11 +4818,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.2" +version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "wasm-bindgen", ] @@ -4802,27 +4878,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -4833,9 +4900,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4843,9 +4910,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4853,48 +4920,26 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.4.2" @@ -4908,23 +4953,11 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.12.1", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -4942,9 +4975,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -5042,7 +5075,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5053,7 +5086,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5342,13 +5375,10 @@ dependencies = [ ] [[package]] -name = "wit-bindgen" -version = "0.51.0" +name = "winnow" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" [[package]] name = "wit-bindgen" @@ -5356,85 +5386,6 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.12.1", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" @@ -5478,7 +5429,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -5498,22 +5449,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5533,15 +5484,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" @@ -5573,7 +5524,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c68ec97..4a63b51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,10 @@ default-members = [ [patch.crates-io] #btracker-fs = { git = "https://github.com/yggverse/btracker.git", package = "btracker-fs" } #btracker-fs = { path = "crates/fs" } +btracker-scrape = { path = "crates/scrape" } #librqbit = { git = "https://github.com/ikatson/rqbit.git", rev="ba07a75c9ad3bbb5e1d8aa8dee815ce8820fe372", package = "librqbit" } #librqbit = { version = "9.0.0-beta.1", path = "../../rqbit/crates/librqbit", package = "librqbit" } -btpeer = { git = "https://codeberg.org/YGGverse/btpeer.git" } -#btpeer = { path = "../../codeberg/yggverse/btpeer" } \ No newline at end of file +#btpeer = { git = "https://codeberg.org/YGGverse/btpeer.git" } +#btpeer = { path = "../../codeberg/yggverse/btpeer" } diff --git a/crates/crawler/Cargo.toml b/crates/crawler/Cargo.toml index d2b61bc..bbe224c 100644 --- a/crates/crawler/Cargo.toml +++ b/crates/crawler/Cargo.toml @@ -1,32 +1,32 @@ [package] name = "btracker-crawler" -version = "0.1.0" +version = "0.3.0" edition = "2024" license = "MIT" readme = "README.md" -description = "SSD-friendly FS crawler of BEP 48 / Full Scrape, based on the librqbit resolver" -keywords = ["full-scrape", "librqbit", "bittorrent", "crawler", "resolver"] +description = "SSD-friendly FS crawler of BEP 48 / Full Scrape, based on librqbit" +keywords = ["bittorrent", "dht", "i2p", "librqbit", "crawler"] categories = ["network-programming"] repository = "https://github.com/YGGverse/btracker" # homepage = "https://yggverse.github.io" -[features] -i2p = ["dep:btpeer", "dep:cyphernet", "dep:yosemite"] - [dependencies] anyhow = "1.0.102" -bendy = "0.6.1" -btpeer = { version = "0.8.0", optional = true } -btracker-fs = { version = "0.3.1", features = ["crawler"] } +btpeer = "0.11.1" +btracker-fs = { version = "0.5.0", features = ["crawler"] } chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive"] } -cyphernet = { version = "0.5.4", features = ["i2p"], optional = true } +cyphernet = { version = "0.5.4", features = ["i2p"] } +data-encoding = "2.11.0" librqbit = { version = "9.0.0-rc.0", features = ["disable-upload"]} log = "0.4.30" +rand = "0.10.1" regex = "1.12.3" -reqwest = "0.13.4" +serde = { version = "1.0.228", features = ["derive"] } +serde-inline-default = "1.0.1" +sha2 = "0.11.0" tokio = { version = "1.52.3", features = ["full"] } +toml = "1.1.2" tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } url = "2.5.8" -urlencoding = "2.1.3" -yosemite = { version = "0.7.0", optional = true } \ No newline at end of file +yosemite = "0.7.0" diff --git a/crates/crawler/README.md b/crates/crawler/README.md index b9dc905..b0a5d3d 100644 --- a/crates/crawler/README.md +++ b/crates/crawler/README.md @@ -3,8 +3,9 @@ [![Dependencies](https://deps.rs/repo/github/YGGverse/btracker-crawler/status.svg)](https://deps.rs/repo/github/YGGverse/btracker-crawler) [![crates.io](https://img.shields.io/crates/v/btracker-crawler.svg)](https://crates.io/crates/btracker-crawler) -SSD-friendly FS crawler of BEP 48 / Full Scrape, based on the [librqbit](https://github.com/ikatson/rqbit/tree/main/crates/librqbit) resolver +SSD-friendly FS crawler of BEP 48 / Full Scrape, based on [librqbit](https://github.com/ikatson/rqbit/tree/main/crates/librqbit) > [!NOTE] +> * See also: [config.toml](https://codeberg.org/YGGverse/server/src/branch/main/etc/btracker/crawler.toml) example > * By using OpenTracker as the index source, please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! -> * Enable experimental I2P support by `cargo build --features i2p` \ No newline at end of file +> * I2P option in development, do not use it on production! \ No newline at end of file diff --git a/crates/crawler/src/config.rs b/crates/crawler/src/config.rs index d500a4e..e7b8a31 100644 --- a/crates/crawler/src/config.rs +++ b/crates/crawler/src/config.rs @@ -1,144 +1,51 @@ -use clap::Parser; -use regex::Regex; -use std::{net::SocketAddr, path::PathBuf}; +mod preload; +mod timeout; +mod tracker; + +use preload::Preload; +use serde::Deserialize; +use serde_inline_default::serde_inline_default; +use std::net::SocketAddr; +use timeout::Timeout; +use tracker::Tracker; use url::Url; -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] +#[serde_inline_default] +#[derive(Deserialize)] pub struct Config { - /// Directory path to store preloaded data (e.g. `.torrent` files) - /// - /// * it's probably the same location as `public` dir for the `btracker-http` frontend - #[arg(long, short)] - pub preload: PathBuf, - - /// Absolute path(s) or URL(s) to the BEP 48 / Full Scrape - #[arg(long, short)] - pub full_scrape: Vec, - - /// How long to wait for I2P tracker full scrape response - /// * by compiling OpenTracker for `/scrape`, - /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! - #[arg(long, default_value_t = 15)] - pub full_scrape_timeout: u64, - - /// Compact long data - #[arg(long, default_value = "gzip")] - pub full_scrape_compression: String, - - /// The P2P Blocklist file URL (to filter outgoing connections) - /// - /// * use `--blocklist=file:///path/to/blocklist.txt` format for the local path - #[arg(long)] - pub blocklist: Option, - - /// Define custom tracker(s) to preload the `.torrent` files info - #[arg(long, short)] - pub tracker: Vec, - - /// Define initial peer(s) to preload the `.torrent` files info - #[arg(long)] - pub initial_peer: Option>, - - /// Appends `--tracker` value to magnets and torrents - #[arg(long, default_value_t = false)] - pub export_trackers: bool, - - /// Enable DHT resolver - #[arg(long, default_value_t = false)] - pub enable_dht: bool, - - /// Enable LSD multicast - #[arg(long, default_value_t = false)] - pub enable_lsd: bool, - - /// Disable TCP connection - #[arg(long, default_value_t = false)] - pub disable_tcp: bool, - - /// Bind resolver session on specified device name (`tun0`, `mycelium`, etc.) - #[arg(long)] - pub bind: Option, - - /// Preload only files match regex pattern (list only without preload by default) - /// * see also `preload_max_filesize`, `preload_max_filecount` options - /// - /// ## Example: - /// - /// Filter by image ext - /// ``` - /// --preload-regex '\.(png|gif|jpeg|jpg|webp|svg|log|nfo|txt)$' - /// ``` - /// - /// * requires `storage` argument defined - #[arg(long)] - pub preload_regex: Option, - - /// Max size sum of preloaded files per torrent (match `preload_regex`) - #[arg(long)] - pub preload_max_filesize: Option, - - /// Max count of preloaded files per torrent (match `preload_regex`) - #[arg(long)] - pub preload_max_filecount: Option, - - /// Limit download speed (b/s) - #[arg(long)] - pub download_limit: Option, // * reminder: upload feature is not planed by the crawler impl - - /// Use `socks5://[username:password@]host:port` - #[arg(long)] - pub proxy_url: Option, + /// Preload data config + pub preload: Preload, /// Estimated info-hash index capacity /// /// * use for memory optimization, depending on tracker volumes - #[arg(long, default_value_t = 1000)] - pub index_capacity: usize, + #[serde_inline_default(1000)] + pub info_hash_capacity: usize, /// Crawl loop delay in seconds - #[arg(long, default_value_t = 60)] - pub sleep: u64, + #[serde_inline_default(60)] + pub sleep_seconds: u64, - /// Skip and ban slow or unresolvable hashes - /// when the specified value in seconds is reached - /// - /// * the ban time is dynamically calculated based on the current ban list collected - /// * tip: increase this value when using I2P features - #[arg(long, default_value_t = 60)] - pub timeout: u64, + /// Tracker settings + pub tracker: Tracker, - /// Special trackers such as [opentracker-i2p](https://github.com/r4sas/opentracker-i2p) - /// that return B32 addresses instead of IP - #[cfg(feature = "i2p")] - #[arg(long)] - pub i2p_tracker: Vec, + /// Bind librqbit session on specified device name (`tun0`, `mycelium`, etc.) + pub bind_device_name: Option, - /// How long to wait for I2P tracker announce response - #[cfg(feature = "i2p")] - #[arg(long, default_value_t = 10)] - pub i2p_tracker_announce_timeout: u64, + /// Limit download speed (b/s) + pub download_limit: Option, - /// Absolute path(s) or URL(s) to the BEP 48 / Full Scrape for i2p_tracker(s) - /// * by compiling OpenTracker for `/scrape`, - /// please make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! - #[cfg(feature = "i2p")] - #[arg(long, short)] - pub i2p_full_scrape: Vec, + /// Define initial peer(s) to preload the `.torrent` files info + pub initial_peers: Option>, - /// How long to wait for I2P tracker full scrape response - #[cfg(feature = "i2p")] - #[arg(long, default_value_t = 10)] - pub i2p_full_scrape_timeout: u64, + /// Use `socks5://[username:password@]host:port` for librqbit connections + pub proxy_url: Option, - /// Compact long data - #[cfg(feature = "i2p")] - #[arg(long, default_value = "gzip")] - pub i2p_full_scrape_compression: String, + /// The P2P Blocklist file URL (to filter outgoing connections) + /// + /// * e.g. `file:///path/to/blocklist.txt` for local file + pub blocklist_url: Option, - /// Use HTTP(s) proxy to resolve `i2p_tracker` and `i2p_full_scrape`, usually `http://127.0.0.1:4444` - /// * skip this setting if the I2P tracker is running locally (for the performance reasons) - #[cfg(feature = "i2p")] - #[arg(long)] - pub i2p_proxy: Option, + #[serde(default)] + pub timeout: Timeout, } diff --git a/crates/crawler/src/config/preload.rs b/crates/crawler/src/config/preload.rs new file mode 100644 index 0000000..c94b27b --- /dev/null +++ b/crates/crawler/src/config/preload.rs @@ -0,0 +1,26 @@ +use serde::Deserialize; +use std::path::PathBuf; + +#[derive(Deserialize)] +pub struct Preload { + /// Directory path to store preloaded data (e.g. `.torrent` files) + /// + /// * it's probably the same location as `public` dir for the `btracker-http` frontend + pub path: PathBuf, + + /// Preload content file (names) match `regex` pattern + /// * see also `max_filesize`, `max_filesize` options + /// + /// ## Example: + /// + /// ``` + /// \.(png|gif|jpeg|jpg|webp|svg|log|nfo|txt)$ + /// ``` + pub regex: Option, + + /// Max size sum of preloaded files per torrent (match `regex`) + pub max_filesize: Option, + + /// Max count of preloaded files per torrent (match `regex`) + pub max_filecount: Option, +} diff --git a/crates/crawler/src/config/timeout.rs b/crates/crawler/src/config/timeout.rs new file mode 100644 index 0000000..1db7360 --- /dev/null +++ b/crates/crawler/src/config/timeout.rs @@ -0,0 +1,34 @@ +use serde::Deserialize; +use serde_inline_default::serde_inline_default; + +const ADD_TORRENT_SECONDS: u64 = 60; +const TORRENT_PRELOAD_SECONDS: u64 = 60; +const I2P_PEER_CONNECT_TIMEOUT_SECONDS: u64 = 60; +const CLEANUP_INACTIVE_I2P_SESSION_SECONDS: u64 = 900; + +#[serde_inline_default] +#[derive(Deserialize)] +pub struct Timeout { + #[serde_inline_default(ADD_TORRENT_SECONDS)] + pub add_torrent_seconds: u64, + + #[serde_inline_default(TORRENT_PRELOAD_SECONDS)] + pub torrent_preload_seconds: u64, + + #[serde_inline_default(CLEANUP_INACTIVE_I2P_SESSION_SECONDS)] + pub cleanup_inactive_i2p_session_seconds: u64, + + #[serde_inline_default(I2P_PEER_CONNECT_TIMEOUT_SECONDS)] + pub i2p_peer_connect_timeout_seconds: u64, +} + +impl Default for Timeout { + fn default() -> Self { + Self { + add_torrent_seconds: ADD_TORRENT_SECONDS, + cleanup_inactive_i2p_session_seconds: CLEANUP_INACTIVE_I2P_SESSION_SECONDS, + i2p_peer_connect_timeout_seconds: I2P_PEER_CONNECT_TIMEOUT_SECONDS, + torrent_preload_seconds: TORRENT_PRELOAD_SECONDS, + } + } +} diff --git a/crates/crawler/src/config/tracker.rs b/crates/crawler/src/config/tracker.rs new file mode 100644 index 0000000..4a20c60 --- /dev/null +++ b/crates/crawler/src/config/tracker.rs @@ -0,0 +1,16 @@ +mod announce; +mod scrape; + +use announce::{Default, I2p}; +use scrape::Scrape; +use serde::Deserialize; + +#[derive(Deserialize)] +pub struct Tracker { + /// Info-hash source + pub scrape: Vec, + + /// Peers source + pub announce: Vec, + pub announce_i2p: Option>, +} diff --git a/crates/crawler/src/config/tracker/announce.rs b/crates/crawler/src/config/tracker/announce.rs new file mode 100644 index 0000000..3965a4a --- /dev/null +++ b/crates/crawler/src/config/tracker/announce.rs @@ -0,0 +1,5 @@ +mod default; +mod i2p; + +pub use default::Default; +pub use i2p::I2p; diff --git a/crates/crawler/src/config/tracker/announce/default.rs b/crates/crawler/src/config/tracker/announce/default.rs new file mode 100644 index 0000000..94a6686 --- /dev/null +++ b/crates/crawler/src/config/tracker/announce/default.rs @@ -0,0 +1,27 @@ +use serde::Deserialize; +use serde_inline_default::serde_inline_default; +use url::Url; + +/// Peers source +#[serde_inline_default] +#[derive(Deserialize)] +pub struct Default { + /// URL to announce + /// + /// * supports HTTP trackers only + pub url: Url, + + /// How long to wait for tracker full scrape response + #[serde_inline_default(5)] + pub timeout: u64, + + /// Static port for outgoing announce connections + #[serde_inline_default(6699)] + pub port: u16, + + /// Use HTTP(s) proxy, e.g. `http://127.0.0.1:9050` or `http://127.0.0.1:4444` for I2P + pub proxy_url: Option, + + /// Max peers per tracker + pub peers_limit: Option, +} diff --git a/crates/crawler/src/config/tracker/announce/i2p.rs b/crates/crawler/src/config/tracker/announce/i2p.rs new file mode 100644 index 0000000..1e1104e --- /dev/null +++ b/crates/crawler/src/config/tracker/announce/i2p.rs @@ -0,0 +1,42 @@ +use serde::Deserialize; +use serde_inline_default::serde_inline_default; +use std::net::{IpAddr, Ipv4Addr}; +use url::Url; + +/// Peers source I2P +#[serde_inline_default] +#[derive(Deserialize)] +pub struct I2p { + /// URL to announce + /// + /// * supports HTTP trackers only + pub url: Url, + + /// How long to wait for tracker full scrape response + #[serde_inline_default(5)] + pub timeout: u64, + + /// Static port for outgoing announce connections + #[serde_inline_default(6699)] + pub port: u16, + + /// Use HTTP(s) proxy, e.g. `http://127.0.0.1:9050` or `http://127.0.0.1:4444` for I2P + pub proxy_url: Option, + + /// Bind I2P / SAM bridge on given host + /// + /// * only if the I2P `full_scrape` trackers in use + #[serde_inline_default(IpAddr::V4(Ipv4Addr::LOCALHOST))] + pub loopback_host: IpAddr, + + /// How many hops do the inbound tunnels of the session have + #[serde_inline_default(3)] + pub inbound_len: usize, + + /// How many hops do the outbound tunnels of the session have + #[serde_inline_default(3)] + pub outbound_len: usize, + + /// Max peers per tracker + pub peers_limit: Option, +} diff --git a/crates/crawler/src/config/tracker/scrape.rs b/crates/crawler/src/config/tracker/scrape.rs new file mode 100644 index 0000000..f3b0f98 --- /dev/null +++ b/crates/crawler/src/config/tracker/scrape.rs @@ -0,0 +1,22 @@ +use serde::Deserialize; +use serde_inline_default::serde_inline_default; +use url::Url; + +/// Info-hash source +/// * tip: by using OpenTracker, +/// make sure `FEATURES+=-DWANT_FULLSCRAPE` is enabled! +#[serde_inline_default] +#[derive(Deserialize)] +pub struct Scrape { + /// URL to the BEP 48 / Full Scrape + /// + /// * supports HTTP trackers only + pub url: Url, + + /// How long to wait for tracker full scrape response + #[serde_inline_default(5)] + pub timeout: u64, + + /// Use HTTP(s) proxy, e.g. `http://127.0.0.1:9050` or `http://127.0.0.1:4444` for I2P + pub proxy_url: Option, +} diff --git a/crates/crawler/src/full_scrape.rs b/crates/crawler/src/full_scrape.rs index c4a982a..903e39a 100644 --- a/crates/crawler/src/full_scrape.rs +++ b/crates/crawler/src/full_scrape.rs @@ -1,90 +1,61 @@ -use anyhow::{Result, bail}; -use bendy::decoding::Decoder; +use anyhow::Result; +use btpeer::{InfoHash, http::query::Scrape}; use librqbit::dht::Id20; use log::*; -use reqwest::{Client, Proxy, header::*}; -use std::{io, time::Duration}; +use std::{collections::HashSet, time::Duration}; +use url::Url; -/// Try parse info-hash from the given source, -/// convert bytes to valid `InfoHash` v1 array on success. -pub async fn get( - source_url: &str, - capacity: usize, - timeout: Duration, - compression_method: &str, - http_proxy_url: Option<&str>, -) -> Result> { - let mut i = Vec::with_capacity(capacity); +pub struct FullScrape { + pub proxy: Option, + pub query: Scrape, + pub timeout: Duration, +} - let bytes = if source_url.starts_with("http") { - trace!("build full scrape request to `{source_url}`..."); - let client = Client::builder().timeout(timeout); - let request = match http_proxy_url { - Some(p) => client.proxy(if p.starts_with("https") { - trace!("applying HTTPs proxy `{p}` to `{source_url}`..."); - Proxy::https(p)? - } else { - trace!("applying HTTP proxy `{p}` to `{source_url}`..."); - Proxy::http(p)? - }), - None => { - trace!("sending direct request to `{source_url}`..."); - client - } - } - .build()? - .get(source_url); +pub struct Buffer(pub Vec); - trace!("sending full scrape request to `{source_url}`..."); - let response = if compression_method.is_empty() { - trace!("disable compression for `{source_url}`..."); - request - } else { - trace!("set `{compression_method}` compression method for `{source_url}`..."); - request.header(ACCEPT_ENCODING, compression_method) - } - .send() - .await?; +impl Buffer { + pub async fn get(&self, expected_capacity: usize) -> Result> { + let mut s = HashSet::with_capacity(expected_capacity); - if !response.status().is_success() { - bail!( - "HTTP request to `{source_url}` failed: {}", - response.status() + for this in self.0.iter() { + let scrape = match btpeer::http::scrape( + &this.query, + this.timeout, + this.proxy.as_ref().map(|u| u.as_str()), ) - } + .await + { + Ok(result) => result, + Err(e) => { + warn!( + "[full-scrape] full-scrape {} update failed: {e}; skip", + &this.query + ); + continue; // skip without panic + } + } + .stats + .into_keys(); - trace!("HTTP response from `{source_url}` received successfully, reading the bytes..."); - let res_bytes = response.bytes().await?; - res_bytes.to_vec() - } else { - trace!("begin full scrape request from `{source_url}`..."); - tokio::fs::read(source_url.trim_start_matches("file://")).await? - }; + let total = scrape.len(); - let mut decoder = Decoder::new(&bytes); + for i in scrape { + s.insert(match i { + InfoHash::V1(ref b) => Id20::from_bytes(b)?, + }); + } - let root_object = decoder - .next_object()? - .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "empty"))?; + debug!( + "[full-scrape] received {total} unique hashes from {}...", + this.query.0 + ) + } - trace!("dictionary parse begin..."); - let mut outer_dict = root_object.try_into_dictionary()?; + debug!( + "[full-scrape] collected {} unique hashes to crawl...", + s.len() + ); - while let Some(pair) = outer_dict.next_pair().unwrap_or(None) { - if pair.0 == b"files" { - trace!("the `files` index found..."); - let mut files_dict = pair - .1 - .try_into_dictionary() - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - while let Some(file_pair) = files_dict.next_pair().unwrap_or(None) { - let id20 = Id20::from_bytes(file_pair.0)?; - trace!("push `{}` to queue...", id20.as_string()); - i.push(id20) - } - break; - } + Ok(s) } - - Ok(i) } diff --git a/crates/crawler/src/i2p.rs b/crates/crawler/src/i2p.rs deleted file mode 100644 index c5443a9..0000000 --- a/crates/crawler/src/i2p.rs +++ /dev/null @@ -1,77 +0,0 @@ -use anyhow::Result; -use log::*; -use std::{collections::HashSet, net::SocketAddr}; -use url::Url; -use yosemite::{Session, style::Stream}; - -/// Create virtual `SocketAddr` interfaces for given I2P trackers -pub async fn get_peers( - id20: &[u8; 20], - trackers: &Vec, - announce_timeout: u64, - trackers_proxy: Option<&Url>, -) -> Result> { - use btpeer::http::{announce, query::Announce}; - let mut peers = HashSet::new(); - for tracker in trackers { - debug!("get peers from I2P tracker `{tracker}`..."); - let mut peers_buffer = btpeer::peer::new_buffer(None); - let a = Announce::new(tracker.as_str(), id20, 0)?; - trace!("sending announce `{a}`..."); - announce( - &a, - std::time::Duration::from_secs(announce_timeout), - trackers_proxy.map(|url| url.as_str()), - Some(&mut peers_buffer), - ) - .await?; - let t = peers_buffer.len(); - trace!("received {t} peers total..."); - for (i, p) in peers_buffer.into_iter().enumerate() { - trace!("handle peer `{p}` ({i}/{t})..."); - match p.host { - cyphernet::addr::HostName::I2p(i2p) => match new_bridge(i2p.to_string()).await { - Ok(peer) => { - let s = peer.to_string(); - if peers.insert(peer) { - trace!("inserting new I2P peer `{s}`...") - } else { - trace!("replacing existing I2P peer `{s}`...") - } - } - Err(e) => warn!("could not create now socket for peer `{i2p}`: `{e}`; skip."), - }, - n => warn!("unexpected I2P address family: `{n}`; skip."), - } - } - } - debug!("collected {} unique peers total.", peers.len()); - trace!("virtual peer bridges returned: {peers:?}"); - Ok(peers) -} - -/// Create new outbound virtual stream to destination, -/// return local`SocketAddr` on success. -/// -/// Destination can be: -/// -/// * hostname such as `host.i2p` -/// * base32-encoded session received such as `lhbd7ojcaiofbfku7ixh47qj537g572zmhdc4oilvugzxdpdghua.b32.i2p/` -/// * base64-encoded string received from, e.g., `yosemite::Session::new` -async fn new_bridge(destination: String) -> Result { - let loopback = std::net::SocketAddrV4::new(std::net::Ipv4Addr::new(127, 0, 0, 1), 0); - - let listener = tokio::net::TcpListener::bind(loopback).await?; - let proxy_address = listener.local_addr()?; - let mut session = Session::::new(Default::default()).await?; - - tokio::spawn(async move { - while let Ok((mut local, _)) = listener.accept().await { - if let Ok(mut remote) = session.connect(&destination).await { - let _ = tokio::io::copy_bidirectional(&mut local, &mut remote).await; - } - } - }); - - Ok(proxy_address) -} diff --git a/crates/crawler/src/main.rs b/crates/crawler/src/main.rs index c55fc2a..793c425 100644 --- a/crates/crawler/src/main.rs +++ b/crates/crawler/src/main.rs @@ -1,25 +1,33 @@ mod config; mod full_scrape; - -#[cfg(feature = "i2p")] -mod i2p; +mod opt; +mod tracker; use anyhow::Result; +use btpeer::http::query::Scrape; +use btracker_fs::crawler::Storage; +use chrono::{Local, Utc}; +use clap::Parser; +use config::Config; +use full_scrape::FullScrape; use librqbit::{ - AddTorrent, AddTorrentOptions, AddTorrentResponse, ConnectionOptions, DhtSessionConfig, - Session, SessionOptions, + AddTorrent, AddTorrentOptions, AddTorrentResponse, ConnectionOptions, Session, SessionOptions, + limits::LimitsConfig, +}; +use log::*; +use opt::Opt; +use regex::Regex; +use std::{ + collections::{HashMap, HashSet}, + num::NonZero, + sync::{Arc, atomic::Ordering}, + time::Duration, }; -use std::{collections::HashSet, num::NonZero, time::Duration}; -use url::Url; +use tokio::{sync::RwLock, time}; +use tracker::Tracker; #[tokio::main] async fn main() -> Result<()> { - use btracker_fs::crawler::Storage; - use chrono::Local; - use clap::Parser; - use config::Config; - use log::*; - use tokio::time; // debug if std::env::var("RUST_LOG").is_ok() { use tracing_subscriber::{EnvFilter, fmt::*}; @@ -36,20 +44,122 @@ async fn main() -> Result<()> { } // init components let time_init = Local::now(); - let config = Config::parse(); + + let opt = Opt::parse(); + let config: Config = toml::from_str(&std::fs::read_to_string(&opt.config).unwrap()).unwrap(); + let preload = Storage::init( - config.preload, - config.preload_regex, - config.preload_max_filecount, - config.preload_max_filesize, + config.preload.path, + config.preload.regex.map(|r| Regex::new(&r).unwrap()), + config.preload.max_filecount, + config.preload.max_filesize, ) .unwrap(); - let mut ban = HashSet::with_capacity(config.index_capacity); - info!("crawler started at {time_init}"); + // init info-hash sources + let mut scrape = Vec::with_capacity(config.tracker.scrape.len()); + + for i in config.tracker.scrape { + if !i.url.scheme().starts_with("http") { + todo!("HTTP trackers only!") + } + info!( + "init full-scrape source {}, proxy: {:?}", + i.url, i.proxy_url + ); + scrape.push(FullScrape { + proxy: i.proxy_url, + query: Scrape::new(i.url.as_str(), None)?, + timeout: Duration::from_secs(i.timeout), + }) + } + + let full_scrape = full_scrape::Buffer(scrape); + + // init trackers (for DHT data preload) + let mut trackers = Vec::with_capacity( + config.tracker.announce.len() + + config + .tracker + .announce_i2p + .as_ref() + .map(|a| a.len()) + .unwrap_or_default(), + ); + + for i in config.tracker.announce { + if !i.url.scheme().starts_with("http") { + todo!("HTTP trackers only!") + } + info!("init tracker {}, proxy: {:?}", i.url, i.proxy_url); + trackers.push(Tracker::Default { + proxy: i.proxy_url, + timeout: Duration::from_secs(i.timeout), + url: i.url, + port: i.port, + peers_limit: i.peers_limit, + }) + } + + // init virtual sockets index for I2P / SAM + let virtual_peers = Arc::new(RwLock::new(HashMap::new())); + // init I2P trackers, if exists + if let Some(a) = config.tracker.announce_i2p { + use yosemite::{Session, SessionOptions}; + for i in a { + if !i.url.scheme().starts_with("http") { + todo!("HTTP trackers only!") + } + info!("init I2P tracker {}, proxy: {:?}", i.url, i.proxy_url); + trackers.push(Tracker::I2p { + loopback: i.loopback_host, + proxy: i.proxy_url, + announce_timeout: Duration::from_secs(i.timeout), + peer_connect_timeout: Duration::from_secs( + config.timeout.i2p_peer_connect_timeout_seconds, + ), + url: i.url, + port: i.port, + peers_limit: i.peers_limit, + sam_session: Arc::new(RwLock::new( + Session::new(SessionOptions { + inbound_len: i.inbound_len, + outbound_len: i.outbound_len, + ..SessionOptions::default() + }) + .await?, + )), + peers_map: virtual_peers.clone(), + }) + } + } + + let tracker = tracker::Buffer(trackers); + + // init ban list to skip unresolvable info-hashes between the queue iterations + let mut ban = HashSet::with_capacity(config.info_hash_capacity); + + // start the crawler + info!("crawler started"); loop { let time_queue = Local::now(); - debug!("queue crawl begin at {time_queue}..."); + debug!("queue crawl begin..."); + + // Cleanup inactive I2P sessions if exists + virtual_peers.write().await.retain(|b32, s| { + if (Utc::now().timestamp() as u64).saturating_sub(s.last_active.load(Ordering::Relaxed)) + > config.timeout.cleanup_inactive_i2p_session_seconds + { + debug!( + "I2P session {b32} is inactive; aborting handler on {}", + s.socket + ); + s.handler.abort(); + false + } else { + true + } + }); // Please, note: // * it's important to start new `Session` inside the crawler loop: @@ -59,171 +169,102 @@ async fn main() -> Result<()> { let session = Session::new_with_opts( preload.root().clone(), SessionOptions { - bind_device_name: config.bind.clone(), - blocklist_url: config.blocklist.as_ref().map(|b| b.to_string()), + bind_device_name: config.bind_device_name.clone(), + blocklist_url: config.blocklist_url.as_ref().map(|b| b.to_string()), listen: None, connect: Some(ConnectionOptions { - enable_tcp: !config.disable_tcp, proxy_url: config.proxy_url.as_ref().map(|u| u.to_string()), ..ConnectionOptions::default() }), - dht: if config.enable_dht { - Some(DhtSessionConfig { - persistence: None, - ..DhtSessionConfig::default() - }) - } else { - None - }, - disable_local_service_discovery: !config.enable_lsd, + dht: None, + disable_local_service_discovery: true, disable_upload: true, fastresume: false, persistence: None, - ratelimits: librqbit::limits::LimitsConfig { + ratelimits: LimitsConfig { download_bps: config.download_limit.and_then(NonZero::new), - ..librqbit::limits::LimitsConfig::default() + ..LimitsConfig::default() }, - trackers: config.tracker.iter().cloned().collect(), + trackers: HashSet::new(), // we're resolving peers manually ..SessionOptions::default() }, ) .await?; // build unique ID index from the multiple info-hash sources - let mut queue = HashSet::with_capacity(config.index_capacity); - for source in &config.full_scrape { - debug!("index source `{source}`..."); - for i in match full_scrape::get( - source, - config.index_capacity, - Duration::from_secs(config.full_scrape_timeout), - &config.full_scrape_compression, - None, - ) - .await - { - Ok(i) => { - debug!("fetch `{}` hashes from `{source}`...", i.len()); - i - } - Err(e) => { - warn!("the full scrape `{source}` update failed: `{e}`; skip."); - continue; // skip without panic - } - } { - queue.insert(i); - } - } - #[cfg(feature = "i2p")] - { - for source in &config.i2p_full_scrape { - debug!("index I2P source `{source}`..."); - for i in match full_scrape::get( - source, - config.index_capacity, - Duration::from_secs(config.i2p_full_scrape_timeout), - &config.i2p_full_scrape_compression, - config.i2p_proxy.as_ref().map(|p| p.as_str()), - ) - .await - { - Ok(i) => { - debug!("fetch `{}` hashes from I2P `{source}`...", i.len()); - i - } - Err(e) => { - warn!("I2P full scrape `{source}` update failed: `{e}`; skip."); - continue; // skip without panic - } - } { - queue.insert(i); - } - } - } - + let queue = full_scrape.get(config.info_hash_capacity).await?; // clean up nonexistent ban entries from the memory pool ban.retain(|i| { let is_retain = queue.contains(i); if !is_retain { debug!( - "remove `{}` from the ban list, as it is no longer available in the source.", + "remove {} from the ban list, as it is no longer available in the source.", i.as_string() ) } is_retain }); - // handle debug!( - "fetched {} unique hashes from {} source, banned: {}.", + "fetched {} unique hashes, banned: {}.", queue.len(), - config.full_scrape.len(), ban.len() ); + + // handle for i in queue { // convert to string once let h = i.as_string(); if preload.contains_torrent(&h)? { - debug!("torrent `{h}` exists, skip."); + debug!("torrent {h} exists, skip."); continue; } + // skip banned entry, remove it from the ban list to retry on the next iteration if ban.remove(&i) { - debug!("torrent `{h}` is banned, skip for this queue."); + debug!("torrent {h} is banned, skip."); continue; } - // init unique peers hash table - info!("resolve `{h}`..."); + debug!("resolve {h}..."); + + // discover unique peers first + let initial_peers = match tracker.peers(&i).await { + Ok(mut peers) => { + if let Some(ref p) = config.initial_peers { + debug!("forcefully extend with {} peers ({p:?})", p.len()); + peers.extend(p); + } + if peers.is_empty() { + debug!("could not find peers for torrent {h}, skip."); + continue; + } else { + let l = peers.len(); + debug!("collected {l} peers for torrent {h}."); + peers + } + } + Err(e) => { + warn!("could not get peers for torrent {h}: {e}, skip."); + continue; + } + }; + + // make sure the list is not empty as unexpected here + assert!(!initial_peers.is_empty()); + // run the crawler in single thread for performance reasons, // use `timeout` argument option to skip the dead connections. match time::timeout( - Duration::from_secs(config.timeout), + Duration::from_secs(config.timeout.add_torrent_seconds), session.add_torrent( - AddTorrent::from_url(magnet( - &h, - if config.tracker.is_empty() { - None - } else { - Some(config.tracker.as_ref()) - }, - )), + AddTorrent::from_url(tracker.magnet(&h)), Some(AddTorrentOptions { paused: true, // continue after `only_files` update overwrite: true, - disable_trackers: config.tracker.is_empty(), - initial_peers: { - #[cfg(feature = "i2p")] - { - let mut peers: HashSet = config - .initial_peer - .as_ref() - .map(|p| p.iter().cloned().collect()) - .unwrap_or_default(); - match i2p::get_peers( - &i.0, - &config.i2p_tracker, - config.i2p_tracker_announce_timeout, - config.i2p_proxy.as_ref(), - ) - .await - { - Ok(p) => peers.extend(p), - Err(e) => warn!("{e}"), - } - if peers.is_empty() { - None - } else { - trace!("Collected {} unique peers to handle", peers.len()); - Some(peers.into_iter().collect()) - } - } - #[cfg(not(feature = "i2p"))] - { - config.initial_peer.clone() - } - }, - list_only: false, + disable_trackers: true, // we're resolving peers manually + initial_peers: Some(initial_peers.into_iter().collect()), + list_only: preload.regex.is_none(), // the destination folder to preload files match `preload_regex` // * e.g. images for audio albums output_folder: preload.tmp_dir(&h, true)?.to_str().map(|s| s.to_string()), @@ -234,13 +275,20 @@ async fn main() -> Result<()> { .await { Ok(r) => match r { + Ok(AddTorrentResponse::ListOnly(l)) => { + assert!(preload.regex.is_none()); + debug!("persist bytes for torrent file {h}..."); + preload.commit(&h, l.torrent_bytes.to_vec(), None)?; + info!("torrent {h} resolved.") + } Ok(AddTorrentResponse::Added(_, mt)) => { + assert!(preload.regex.is_some()); assert!(mt.is_paused()); let mut keep_files = HashSet::with_capacity( - config.preload_max_filecount.unwrap_or_default(), + config.preload.max_filecount.unwrap_or_default(), ); let mut only_files = HashSet::with_capacity( - config.preload_max_filecount.unwrap_or_default(), + config.preload.max_filecount.unwrap_or_default(), ); mt.wait_until_initialized().await?; let bytes = mt.with_metadata(|m| { @@ -250,7 +298,7 @@ async fn main() -> Result<()> { .is_some_and(|limit| only_files.len() + 1 > limit) { debug!( - "file count limit ({}) reached, skip file `{id}` for `{h}` at `{}` (and other files after it)", + "file count limit ({}) reached, skip file {id} for {h} at {} (and other files after it)", only_files.len(), info.relative_filename.to_string_lossy() ); @@ -258,21 +306,21 @@ async fn main() -> Result<()> { } if preload.max_filesize.is_some_and(|limit| info.len > limit) { debug!( - "file size ({}) limit reached, skip file `{id}` for `{h}` at `{}`", + "file size ({}) limit reached, skip file {id} for {h} at {}", info.len, info.relative_filename.to_string_lossy() ); continue; } - if preload.regex.as_ref().is_none_or(|r| { + if preload.regex.as_ref().is_some_and(|r| { !r.is_match(&info.relative_filename.to_string_lossy()) }) { - debug!("regex filter, skip file `{id}` for `{h}` at `{}`", + debug!("regex filter match: skip {id} for {h} at {}", info.relative_filename.to_string_lossy()); continue; } debug!( - "keep file `{id}` for `{h}` as `{}`", + "keep file {id} for {h} as {}", info.relative_filename.to_string_lossy() ); assert!(keep_files.insert(info.relative_filename.clone())); @@ -282,42 +330,40 @@ async fn main() -> Result<()> { })?; session.update_only_files(&mt, &only_files).await?; session.unpause(&mt).await?; - debug!("begin torrent `{h}` preload..."); + debug!("begin torrent {h} preload..."); if let Err(e) = time::timeout( - Duration::from_secs(config.timeout), + Duration::from_secs(config.timeout.torrent_preload_seconds), mt.wait_until_completed(), ) .await { - info!( - "skip awaiting the completion of preload torrent data for `{h}` (`{e}`), ban for the next queue.", - ); + debug!("preload torrent data for {h} failed ({e}), ban temporarily."); assert!(ban.insert(i)); continue; } - debug!("torrent `{h}` preload completed."); + debug!("torrent {h} preload completed."); // persist torrent bytes and preloaded content, // cleanup tmp (see rqbit#408) - debug!("persist torrent `{h}` with `{}` files...", keep_files.len()); + debug!("persist torrent {h} with {} files...", keep_files.len()); preload.commit(&h, bytes, Some(keep_files))?; - info!("torrent `{h}` resolved.") + info!("torrent {h} resolved.") } - Ok(_) => panic!(), + Ok(_) => unreachable!(), Err(e) => { - warn!("failed to resolve torrent `{h}`: `{e}`, ban for the next queue."); + debug!("failed to resolve torrent {h}: {e}, ban temporarily."); assert!(ban.insert(i)) } }, Err(e) => { - info!( - "skip awaiting the completion of adding torrent `{h}` (`{e}`), ban for the next queue." + debug!( + "skip awaiting the completion of adding torrent {h} ({e}), ban temporarily." ); assert!(ban.insert(i)) } } } session.stop().await; - info!( + debug!( "queue completed at {time_queue} (time: {} / uptime: {} / banned: {}) await {} seconds to continue...", Local::now() .signed_duration_since(time_queue) @@ -326,20 +372,8 @@ async fn main() -> Result<()> { .signed_duration_since(time_init) .as_seconds_f32(), ban.len(), - config.sleep + config.sleep_seconds ); - std::thread::sleep(Duration::from_secs(config.sleep)) - } -} - -/// Build magnet URI (`librqbit` impl dependency) -fn magnet(info_hash: &str, trackers: Option<&Vec>) -> String { - let mut m = format!("magnet:?xt=urn:btih:{info_hash}"); - if let Some(t) = trackers { - for tracker in t { - m.push_str("&tr="); - m.push_str(&urlencoding::encode(tracker.as_str())) - } + std::thread::sleep(Duration::from_secs(config.sleep_seconds)) } - m } diff --git a/crates/crawler/src/opt.rs b/crates/crawler/src/opt.rs new file mode 100644 index 0000000..71b45ec --- /dev/null +++ b/crates/crawler/src/opt.rs @@ -0,0 +1,10 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Opt { + /// Path to `config.toml` + #[arg(long, short)] + pub config: PathBuf, +} diff --git a/crates/crawler/src/tracker.rs b/crates/crawler/src/tracker.rs new file mode 100644 index 0000000..80dc3f8 --- /dev/null +++ b/crates/crawler/src/tracker.rs @@ -0,0 +1,304 @@ +use anyhow::Result; +use btpeer::Peer; +use chrono::Utc; +use librqbit::dht::Id20; +use log::*; +use std::{ + collections::{HashMap, HashSet}, + net::{IpAddr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; +use tokio::sync::RwLock; +use url::Url; +use yosemite::{Session, style::Stream}; + +pub enum Tracker { + Default { + peers_limit: Option, + port: u16, + proxy: Option, + timeout: Duration, + url: Url, + }, + I2p { + announce_timeout: Duration, + loopback: IpAddr, + peer_connect_timeout: Duration, + peers_limit: Option, + peers_map: Arc>>, + port: u16, + proxy: Option, + sam_session: Arc>>, + url: Url, + }, +} + +impl Tracker { + async fn peers(&self, info_hash: &Id20) -> Result> { + Ok(match self { + Self::Default { + peers_limit, + port, + proxy, + timeout, + url, + } => { + let announce = + btpeer::http::query::Announce::new(url.as_str(), &info_hash.0, *port)?; + + let peers = take_random_peers( + btpeer::http::announce(&announce, *timeout, proxy.as_ref().map(|u| u.as_str())) + .await? + .peers + .0 + .into_iter() + .filter(|p| match p { + Peer::Default(this) => { + url.host_str() + .is_some_and(|h| !h.contains(&this.host.to_string())) + && this.port != *port // exclude self + } + Peer::I2p(..) => false, + }) + .collect(), + *peers_limit, + ); + + let mut b = HashSet::with_capacity(peers.len()); + + for p in peers { + match p { + Peer::Default(peer) => { + let p = SocketAddr::new(peer.host, peer.port); + if b.insert(p) { + debug!("[tracker] add peer: {p}") + } else { + debug!("[tracker] replace existing peer: {p}") + } + } + Peer::I2p(peer) => { + unreachable!( + "[tracker] unexpected peer {peer} from tracker {url}, skip" + ) + } + } + } + b + } + Self::I2p { + announce_timeout, + loopback, + peer_connect_timeout, + peers_limit, + peers_map, + port, + proxy, + sam_session, + url, + } => { + let announce = + btpeer::http::query::Announce::new(url.as_str(), &info_hash.0, *port)?; + + let b32 = b32(sam_session.read().await.destination().as_bytes()); + + let peers = take_random_peers( + btpeer::http::announce_i2p( + &announce, + *announce_timeout, + proxy.as_ref().map(|u| u.as_str()), + ) + .await? + .peers + .0 + .into_iter() + .filter(|p| match p { + Peer::I2p(this) => this.b32 != b32, // exclude self + Peer::Default(..) => false, + }) + .collect(), + *peers_limit, + ); + + let mut m = peers_map.write().await; // prevents infinitive async socket spawn + let mut b = HashSet::with_capacity(peers.len()); // resulting peers buffer + + for p in peers { + match p { + Peer::I2p(peer) => { + if let Some(i2p_session) = m.get(&peer.b32) { + i2p_session + .last_active + .store(Utc::now().timestamp() as u64, Ordering::Relaxed); + + b.insert(i2p_session.socket); + debug!( + "[tracker] reuse existing I2P peer {peer} as {}", + i2p_session.socket + ); + continue; + } + + debug!("[tracker] init SAM proxy for {peer} on {loopback}..."); + + let listener = + tokio::net::TcpListener::bind(SocketAddr::new(*loopback, 0)) + .await?; + + let socket = listener.local_addr()?; + + if b.insert(socket) { + debug!("[tracker] bind I2P peer {peer} as {socket}") + } else { + debug!("[tracker] bind existing I2P peer {peer} as {socket}") + } + + debug!( + "[tracker] listening incoming connections for {peer} on {socket} as {b32}...", + ); + + let timeout = *peer_connect_timeout; + let session = sam_session.clone(); + let peer_b32 = peer.b32.clone(); + let handler = tokio::spawn(async move { + while let Ok((mut local, client)) = listener.accept().await { + debug!( + "[tracker] accepting SAM connection from {client} ({peer_b32})" + ); + match tokio::time::timeout( + timeout, + session.write().await.connect(&peer_b32), + ) + .await + { + Ok(connection) => match connection { + Ok(mut remote) => { + debug!( + "[tracker] begin SAM connection to {}", + remote.remote_destination() // | &peer_b32 + ); + match tokio::io::copy_bidirectional( + &mut local, + &mut remote, + ) + .await // @TODO timeout? + { + Ok((a, b)) => trace!( + "[tracker] copied {a}/{b} to {}", + remote.remote_destination() // | &peer_b32 + ), + Err(e) => debug!("{e}"), + } + } + Err(e) => debug!( + "[tracker] connection failed to {client} ({peer_b32}): {e}" + ), + }, + Err(e) => debug!( + "[tracker] connection to {client} ({peer_b32}) timed out after {} seconds: {e}", + timeout.as_secs() + ), + } + } + }); + assert!( + m.insert( + peer.b32, + I2pSession { + socket, + handler, + last_active: AtomicU64::new(Utc::now().timestamp() as u64), + }, + ) + .is_none() + ) + } + Peer::Default(peer) => { + warn!("[tracker] unexpected peer {peer} from I2P tracker {url}, skip") + } + } + } + b + } + }) + } + + fn url(&self) -> &Url { + match self { + Self::Default { url, .. } => url, + Self::I2p { url, .. } => url, + } + } +} + +pub struct I2pSession { + pub handler: tokio::task::JoinHandle<()>, + pub last_active: AtomicU64, + pub socket: SocketAddr, +} + +pub struct Buffer(pub Vec); + +impl Buffer { + /// Return peers from trackers + pub async fn peers(&self, info_hash: &Id20) -> Result> { + let mut peers = HashSet::new(); // unique peers buffer collected from all trackers + + for tracker in self.0.iter() { + debug!( + "[tracker] get peers from {} for {}...", + tracker.url(), + info_hash.as_string(), + ); + peers.extend(tracker.peers(info_hash).await?) + } + + Ok(peers) + } + + /// Build magnet URI (`librqbit` impl dependency) + pub fn magnet(&self, info_hash: &str) -> String { + format!("magnet:?xt=urn:btih:{info_hash}") + } +} + +fn take_random_peers(mut peers: Vec, limit: Option) -> Vec { + use rand::seq::SliceRandom; + + let total = peers.len(); + + let mut rng = rand::rng(); + peers.shuffle(&mut rng); + + match limit { + Some(l) => { + let p: Vec = peers.into_iter().take(l).collect(); + debug!( + "[tracker] taken random peers: {}/{total} (limited to {l} max)", + p.len() + ); + p + } + None => { + debug!("[tracker] taken random peers: {total}"); + peers + } + } +} + +fn b32(destination: &[u8]) -> String { + use data_encoding::BASE32_NOPAD; + use sha2::{Digest, Sha256}; + + let mut hasher = Sha256::new(); + hasher.update(destination); + let hash_result = hasher.finalize(); + + format!( + "{}.b32.i2p", + BASE32_NOPAD.encode(&hash_result).to_lowercase() + ) +} diff --git a/crates/fs/Cargo.toml b/crates/fs/Cargo.toml index d5cb856..95887d3 100644 --- a/crates/fs/Cargo.toml +++ b/crates/fs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "btracker-fs" -version = "0.4.0" +version = "0.5.0" edition = "2024" license = "MIT" readme = "README.md" @@ -16,7 +16,7 @@ public = ["dep:chrono", "dep:librqbit-core"] crawler = ["dep:log", "dep:regex"] [dependencies] -chrono = { version = "0.4.41", features = ["serde"], optional = true } +chrono = { version = "0.4.45", features = ["serde"], optional = true } librqbit-core = { version = "5.0.0", optional = true } log = { version = "0.4.28", optional = true } regex = { version = "1.11.2", optional = true } diff --git a/crates/fs/src/crawler.rs b/crates/fs/src/crawler.rs index 03388f3..beab01e 100644 --- a/crates/fs/src/crawler.rs +++ b/crates/fs/src/crawler.rs @@ -124,7 +124,7 @@ impl Storage { /// Check the given hash is contain resolved torrent file pub fn contains_torrent(&self, info_hash: &str) -> Result { - Ok(fs::exists(self.torrent(info_hash))?) + fs::exists(self.torrent(info_hash)) } /// Get absolute path to the torrent file diff --git a/crates/fs/src/public.rs b/crates/fs/src/public.rs index 1155ce9..94261f8 100644 --- a/crates/fs/src/public.rs +++ b/crates/fs/src/public.rs @@ -7,6 +7,7 @@ use chrono::{DateTime, Utc}; use librqbit_core::Id20; use std::{ fs, + future::Future, io::Error, path::{Path, PathBuf}, str::FromStr, @@ -66,32 +67,37 @@ impl Storage { }) } - pub fn torrents( + pub async fn torrents( &self, keyword: Option<&str>, sort_order: Option<(Sort, Order)>, start: Option, limit: Option, - visibility_filter: impl Fn(Id20) -> bool, - ) -> Result { + visibility_filter: F, + ) -> Result + where + F: Fn(Id20) -> Fut, + Fut: Future, + { let f = self.files(keyword, sort_order)?; let t = f.len(); // total let l = limit.unwrap_or(t); let s = start.unwrap_or_default(); let mut b = Vec::with_capacity(l); let mut i = 0; // start offset - for file in f.iter().filter(|file| { - file.path - .file_stem() - .is_some_and(|n| Id20::from_str(&n.to_string_lossy()).is_ok_and(&visibility_filter)) - }) { - if i >= s && b.len() < l { - b.push(Torrent { - bytes: fs::read(&file.path)?, - time: file.modified.into(), - }) + for file in f.iter() { + if let Some(n) = file.path.file_stem() + && let Ok(id20) = Id20::from_str(&n.to_string_lossy()) + && visibility_filter(id20).await + { + if i >= s && b.len() < l { + b.push(Torrent { + bytes: fs::read(&file.path)?, + time: file.modified.into(), + }); + } + i += 1; } - i += 1 } Ok(Torrents { total: t, diff --git a/crates/gemini/Cargo.toml b/crates/gemini/Cargo.toml index bbdb501..4e2cfac 100644 --- a/crates/gemini/Cargo.toml +++ b/crates/gemini/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "btracker-gemini" -version = "0.2.0" +version = "0.3.0" edition = "2024" license = "MIT" readme = "README.md" @@ -11,16 +11,17 @@ repository = "https://github.com/YGGverse/btracker" [dependencies] anyhow = "1.0.95" -btracker-fs = { version = "0.4.0", features = ["public"] } -btracker-scrape = "0.1.0" -chrono = "^0.4.20" +btracker-fs = { version = "0.5.0", features = ["public"] } +btracker-scrape = "0.2.0" +chrono = "0.4.45" clap = { version = "4.5.30", features = ["derive"] } +librqbit-core = "5.0.0" log = "0.4.28" native-tls = "0.2.14" +plurify = "0.2.0" +regex = "1.11.2" titanite = "0.3.2" +tokio = { version = "1.52.3", features = ["full"] } tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } -librqbit-core = "5.0.0" -plurify = "0.2.0" url = "2.5.7" -urlencoding = "2.1.3" -regex = "1.11.2" \ No newline at end of file +urlencoding = "2.1.3" \ No newline at end of file diff --git a/crates/gemini/src/config.rs b/crates/gemini/src/config.rs index b841e57..8495620 100644 --- a/crates/gemini/src/config.rs +++ b/crates/gemini/src/config.rs @@ -1,6 +1,6 @@ use clap::Parser; use std::{ - net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, path::PathBuf, }; use url::Url; @@ -25,8 +25,20 @@ pub struct Config { pub tracker: Option>, /// Scrape(s) to local peers count resolve - #[arg(short, long)] - pub scrape: Option>, + #[arg(long)] + pub scrape: Vec, + + /// Timeout to wait for scrape response + #[arg(long, default_value_t = 1)] + pub scrape_timeout: u64, + + /// Proxy for scrape requests + #[arg(long)] + pub scrape_proxy: Option, + + /// Proxy for I2P `tracker` scrape requests + #[arg(long)] + pub scrape_proxy_i2p: Option, /// Bind server `host:port` to listen incoming connections on it #[arg(short, long, default_value_t = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 1965)))] @@ -41,7 +53,7 @@ pub struct Config { pub password: String, /// btracker-fs directory - #[arg(short, long)] + #[arg(short = 'S', long)] pub storage: PathBuf, /// Listing items limit @@ -51,13 +63,4 @@ pub struct Config { /// Default index capacity #[arg(short, long, default_value_t = 1000)] pub capacity: usize, - - /// Bind scrape UDP server - /// - /// * requires `tracker` value(s) to enable scrape features - #[arg(long, default_values_t = vec![ - SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), - SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, 0, 0, 0)) - ])] - pub udp: Vec, } diff --git a/crates/gemini/src/main.rs b/crates/gemini/src/main.rs index 7068c3d..1a74bf6 100644 --- a/crates/gemini/src/main.rs +++ b/crates/gemini/src/main.rs @@ -4,21 +4,24 @@ mod route; use anyhow::Result; use btracker_fs::public::{Order, Sort, Storage, Torrent}; -use btracker_scrape::*; +use btracker_scrape::Buffer as Scrape; use config::Config; use librqbit_core::torrent_metainfo::{TorrentMetaV1Owned, torrent_from_bytes}; use log::*; use native_tls::{HandshakeError, Identity, TlsAcceptor, TlsStream}; use std::{ + collections::HashMap, fs::File, io::{Read, Write}, - net::{IpAddr, SocketAddr, TcpListener, TcpStream}, + net::{SocketAddr, TcpListener, TcpStream}, path::PathBuf, sync::Arc, thread, }; +use tokio::sync::RwLock; -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { use chrono::Local; use clap::Parser; @@ -39,35 +42,13 @@ fn main() -> Result<()> { let config = Config::parse(); let state = Arc::new(State { public: Storage::init(&config.storage, config.limit, config.capacity).unwrap(), - scrape: Scrape::init( - config - .scrape - .as_ref() - .map(|u| { - u.iter() - .map(|url| { - use std::str::FromStr; - if url.scheme() == "tcp" { - todo!("TCP scrape is not implemented") - } - if url.scheme() != "udp" { - todo!("Scheme `{}` is not supported", url.scheme()) - } - SocketAddr::new( - IpAddr::from_str( - url.host_str() - .expect("Required valid host value") - .trim_start_matches('[') - .trim_end_matches(']'), - ) - .unwrap(), - url.port().expect("Required valid port value"), - ) - }) - .collect() - }) - .map(|a| (config.udp, a)), - ), + scrape: Scrape::new( + config.scrape, + config.scrape_timeout, + config.scrape_proxy.as_ref(), + config.scrape_proxy_i2p.as_ref(), + ) + .unwrap(), format_date: config.format_date, name: config.name, description: config.description, @@ -104,7 +85,7 @@ fn main() -> Result<()> { Ok(()) } -fn handle( +async fn handle( state: Arc, peer: SocketAddr, connection: Result, HandshakeError>, @@ -144,7 +125,7 @@ fn handle( if header_buffer.last().is_some_and(|&b| b == b'\n') { // header bytes contain valid Gemini **request** if let Ok(request) = request::Gemini::from_bytes(&header_buffer) { - return response(request, &state, &peer, &mut stream); + return response(request, &state, &peer, &mut stream).await; } // header bytes received but yet could not be parsed, @@ -180,7 +161,7 @@ fn handle( } } -fn response( +async fn response( request: titanite::request::Gemini, state: &State, peer: &SocketAddr, @@ -219,7 +200,7 @@ fn response( }, } .into_bytes(), - Route::List { page, keyword } => match list(state, keyword.as_deref(), page) { + Route::List { page, keyword } => match list(state, keyword.as_deref(), page).await { Ok(data) => success::Default { data: data.as_bytes(), meta: success::default::Meta { @@ -240,7 +221,7 @@ fn response( }) .into_bytes(), Route::Info(id) => match state.public.torrent(id) { - Some(torrent) => match info(state, torrent) { + Some(torrent) => match info(state, torrent).await { Ok(data) => success::Default { data: data.as_bytes(), meta: success::default::Meta { @@ -302,33 +283,41 @@ fn send(data: &[u8], stream: &mut TlsStream, callback: impl FnOnce(Re })()); } -fn list(state: &State, keyword: Option<&str>, page: Option) -> Result { - use std::{cell::RefCell, collections::HashMap, rc::Rc}; - +async fn list(state: &State, keyword: Option<&str>, page: Option) -> Result { /// format search keyword as the pagination query fn query(keyword: Option<&str>) -> String { keyword.map(|k| format!("?{}", k)).unwrap_or_default() } - let scrape_index: Rc>> = - Rc::new(RefCell::new(HashMap::new())); // scrape info-hashes once - - let result = state.public.torrents( - keyword, - Some((Sort::Modified, Order::Desc)), - page.map(|p| if p > 0 { p - 1 } else { p } * state.public.default_limit), - Some(state.public.default_limit), - { - let si = scrape_index.clone(); - move |id| { - state.scrape.get(id.0).is_none_or(|s| { - let is_active = s.leechers > 0 || s.peers > 0 || s.seeders > 0; - assert!(si.borrow_mut().insert(id, s).is_none()); - keyword.is_some() || is_active // show all torrents on search request or hide inactive - }) - } - }, - )?; + let scrape_index: Arc>> = + Arc::new(RwLock::new(HashMap::new())); // scrape info-hashes once + + let result = state + .public + .torrents( + keyword, + Some((Sort::Modified, Order::Desc)), + page.map(|p| if p > 0 { p - 1 } else { p } * state.public.default_limit), + Some(state.public.default_limit), + { + let si = scrape_index.clone(); + let keyword_exists = keyword.is_some(); + + move |id20| { + let si = si.clone(); + async move { + if let Ok(s) = state.scrape.get(&[id20.0]).await { + let is_active = s.incomplete > 0 || s.downloaded > 0 || s.complete > 0; + assert!(si.write().await.insert(id20.0, s).is_none()); + keyword_exists || is_active + } else { + keyword_exists + } + } + } + }, + ) + .await?; let mut b = Vec::new(); @@ -367,7 +356,7 @@ fn list(state: &State, keyword: Option<&str>, page: Option) -> Result, page: Option) -> Result, page: Option) -> Result Result { +async fn info(state: &State, torrent: Torrent) -> Result { struct File { path: Option, length: u64, @@ -463,16 +455,15 @@ fn info(state: &State, torrent: Torrent) -> Result { state.name )); + let t = state.scrape.get(&[i.info_hash.0]).await.unwrap_or_default(); b.push(format!( - "{} • {} • {}{}\n", + "{} • {} • {} • ↑ {} ↓ {} ⏲ {}\n", torrent.time.format(&state.format_date), format::total(&i), format::files(&i), - state - .scrape - .get(i.info_hash.0) - .map(|s| format!(" • ↑ {} ↓ {} ⏲ {}", s.seeders, s.peers, s.leechers)) - .unwrap_or_default() + t.complete, + t.downloaded, + t.incomplete )); b.push(format!( diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index 8b7dc13..38903bb 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "btracker-http" -version = "0.1.1" +version = "0.2.0" edition = "2024" license = "MIT" readme = "README.md" @@ -11,8 +11,9 @@ repository = "https://github.com/yggverse/btracker" # homepage = "https://yggverse.github.io" [dependencies] -btracker-fs = { version = "0.4.0", features = ["public"] } -btracker-scrape = { version = "0.1.0" } +btpeer = "0.11.1" +btracker-fs = { version = "0.5.0", features = ["public"] } +btracker-scrape = "0.2.0" chrono = { version = "0.4.44", features = ["serde"] } clap = { version = "4.6.1", features = ["derive"] } librqbit-core = "5.0.0" diff --git a/crates/http/README.md b/crates/http/README.md index 850bbdf..0b8dc96 100644 --- a/crates/http/README.md +++ b/crates/http/README.md @@ -42,9 +42,7 @@ See the [Wiki](https://github.com/YGGverse/btracker/wiki/Screenshots) page * [ ] announce * [ ] announce list * [x] Common features - * [x] scrape peers/seeders/leechers (by the [btracker-scrape](https://github.com/yggverse/btracker-scrape)) - * [x] UDP - * [ ] TCP + * [x] scrape peers/seeders/leechers * [x] download * [x] magnet link * [x] torrent file diff --git a/crates/http/src/config.rs b/crates/http/src/config.rs index cf46671..16153ab 100644 --- a/crates/http/src/config.rs +++ b/crates/http/src/config.rs @@ -1,6 +1,6 @@ use clap::Parser; use std::{ - net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + net::{IpAddr, Ipv4Addr}, path::PathBuf, }; use url::Url; @@ -59,20 +59,21 @@ pub struct Config { #[arg(long, short, default_value_t = 8000)] pub port: u16, - /// Bind local UDP socket on given address - /// - /// * the default UDP server is not in use without the optional `scrape` argument value - #[arg(long, default_values_t = vec![ - SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), - SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, 0, 0, 0)) - ])] - pub udp: Vec, + /// Scrape(s) to local peers count resolve + #[arg(long)] + pub scrape: Vec, - /// Scrape given trackers (to display peers/seeders/leechers info) - /// - /// * supports multi-stack IPv4/IPv6 trackers + /// Timeout to wait for scrape response + #[arg(long, default_value_t = 1)] + pub scrape_timeout: u64, + + /// Proxy for scrape requests + #[arg(long)] + pub scrape_proxy: Option, + + /// Proxy for I2P `tracker` scrape requests #[arg(long)] - pub scrape: Option>, + pub scrape_proxy_i2p: Option, /// Configure instance in the debug mode #[arg(long, default_value_t = false)] diff --git a/crates/http/src/info_hash.rs b/crates/http/src/info_hash.rs index 3a11a24..746d9b1 100644 --- a/crates/http/src/info_hash.rs +++ b/crates/http/src/info_hash.rs @@ -27,9 +27,6 @@ impl InfoHash { pub fn id20(&self) -> Id20 { self.0 } - pub fn bytes20(&self) -> [u8; 20] { - self.0.0 - } } impl<'r> rocket::request::FromParam<'r> for InfoHash { diff --git a/crates/http/src/main.rs b/crates/http/src/main.rs index 9a62ac4..1c42da4 100644 --- a/crates/http/src/main.rs +++ b/crates/http/src/main.rs @@ -5,11 +5,12 @@ mod config; mod feed; mod info_hash; mod meta; -mod scrape; mod torrent; +use std::sync::Arc; + use btracker_fs::public::{Order, Sort, Storage}; -use btracker_scrape::Scrape; +use btracker_scrape::Buffer as Scrape; use config::Config; use feed::Feed; use info_hash::InfoHash; @@ -19,19 +20,28 @@ use rocket::{ http::{ContentType, Header, Status}, response::{Responder, Response, content::RawXml}, serde::Serialize, + tokio::sync::RwLock, }; use rocket_dyn_templates::{Template, context}; use torrent::Torrent; #[get("/?&")] -fn index( +async fn index( search: Option<&str>, page: Option, scrape: &State, storage: &State, meta: &State, ) -> Result { - use std::{cell::RefCell, collections::HashMap, rc::Rc}; + use std::collections::HashMap; + + #[derive(rocket::serde::Serialize, Default)] + #[serde(crate = "rocket::serde")] + pub struct S { + pub leechers: u32, + pub peers: u32, + pub seeders: u32, + } #[derive(Serialize)] #[serde(crate = "rocket::serde")] @@ -41,13 +51,13 @@ fn index( indexed: String, magnet: String, torrent: String, - scrape: Option, + scrape: Option, size: String, this: Torrent, } - let scrape_index: Rc>> = - Rc::new(RefCell::new(HashMap::new())); // scrape info-hashes once + let scrape_index: Arc>> = + Arc::new(RwLock::new(HashMap::new())); // scrape info-hashes once let result = storage .torrents( @@ -57,20 +67,62 @@ fn index( Some(storage.default_limit), { let si = scrape_index.clone(); - move |id| { - scrape::get(scrape, id.0).is_none_or(|s| { - let is_active = s.leechers > 0 || s.peers > 0 || s.seeders > 0; - assert!(si.borrow_mut().insert(id, s).is_none()); - search.is_some() || is_active - }) + let is_search = search.is_some(); + + move |id20| { + let si = si.clone(); + async move { + if let Ok(s) = scrape.get(&[id20.0]).await { + let is_active = s.incomplete > 0 || s.downloaded > 0 || s.complete > 0; + assert!( + si.write() + .await + .insert( + id20, + S { + leechers: s.incomplete, + peers: s.downloaded, + seeders: s.complete + } + ) + .is_none() + ); + is_search || is_active + } else { + is_search + } + } } }, ) + .await .map_err(|e| { error!("Torrents public storage read error: `{e}`"); Status::InternalServerError })?; + let mut rows = Vec::with_capacity(result.list.len()); + { + let mut scrape_lock = scrape_index.write().await; + for t in result.list.into_iter() { + match Torrent::from_public(&t.bytes, t.time) { + Ok(this) => rows.push(R { + created: this + .creation_date + .map(|t| t.format(&meta.format_time).to_string()), + files: this.files(), + indexed: this.time.format(&meta.format_time).to_string(), + magnet: this.magnet(meta.trackers.as_ref()), + torrent: this.torrent(), // @TODO customize trackers + scrape: scrape_lock.remove(&this.id), + size: this.size(), + this, + }), + Err(e) => error!("Torrent storage read error: `{e}`"), + } + } + } + Ok(Template::render( "index", context! { @@ -98,37 +150,19 @@ fn index( back: page.map(|p| uri!(index(search, if p > 2 { Some(p - 1) } else { None }))), next: if page.unwrap_or(1) * storage.default_limit >= result.visible { None } else { Some(uri!(index(search, Some(page.map_or(2, |p| p + 1))))) }, - rows: result.list - .into_iter() - .filter_map(|t| match Torrent::from_public(&t.bytes, t.time) { - Ok(this) => Some(R { - created: this.creation_date.map(|t| t.format(&meta.format_time).to_string()), - files: this.files(), - indexed: this.time.format(&meta.format_time).to_string(), - magnet: this.magnet(meta.trackers.as_ref()), - torrent: this.torrent(), // @TODO customize trackers - scrape: scrape_index.borrow_mut().remove(&this.id), - size: this.size(), - this - }), - Err(e) => { - error!("Torrent storage read error: `{e}`"); - None - } - }) - .collect::>(), page: page.unwrap_or(1), pages: (result.visible as f64 / storage.default_limit as f64).ceil(), total: result.total, visible: result.visible, is_search: search.is_some(), - search + search, + rows }, )) } #[get("/", rank = 1)] -fn info( +async fn info( info_hash: InfoHash, storage: &State, scrape: &State, @@ -136,6 +170,13 @@ fn info( ) -> Result { match storage.torrent(info_hash.id20()) { Some(t) => { + #[derive(rocket::serde::Serialize, Default)] + #[serde(crate = "rocket::serde")] + pub struct S { + pub leechers: u32, + pub peers: u32, + pub seeders: u32, + } #[derive(Serialize)] #[serde(crate = "rocket::serde")] struct F { @@ -177,7 +218,11 @@ fn info( indexed: this.time.format(&meta.format_time).to_string(), magnet: this.magnet(meta.trackers.as_ref()), torrent: this.torrent(), // @TODO customize trackers - scrape: scrape::get(scrape, info_hash.bytes20()), + scrape: scrape.get(&[info_hash.id20().0]).await.ok().map(|s| S { + leechers: s.incomplete, + peers: s.downloaded, + seeders: s.complete + }), size: this.size(), this }, @@ -262,7 +307,7 @@ fn torrent_file( } #[get("/rss")] -fn rss( +async fn rss( meta: &State, scrape: &State, storage: &State, @@ -279,11 +324,14 @@ fn rss( Some((Sort::Modified, Order::Desc)), None, Some(storage.default_limit), - |id| { - scrape::get(scrape, id.0) - .is_some_and(|s| s.leechers > 0 || s.peers > 0 || s.seeders > 0) + async move |id| { + scrape + .get(&[id.0]) + .await + .is_ok_and(|s| s.incomplete > 0 || s.downloaded > 0 || s.complete > 0) }, ) + .await .map_err(|e| { error!("Torrent storage read error: `{e}`"); Status::InternalServerError @@ -305,34 +353,6 @@ fn rocket() -> _ { if config.canonical_url.is_none() { warn!("Canonical URL option is required for the RSS feed by the specification!") // @TODO } - let scrape = Scrape::init( - config - .scrape - .map(|u| { - u.into_iter() - .map(|url| { - use std::str::FromStr; - if url.scheme() == "tcp" { - todo!("TCP scrape is not implemented") - } - if url.scheme() != "udp" { - todo!("Scheme `{}` is not supported", url.scheme()) - } - std::net::SocketAddr::new( - std::net::IpAddr::from_str( - url.host_str() - .expect("Required valid host value") - .trim_start_matches('[') - .trim_end_matches(']'), - ) - .unwrap(), - url.port().expect("Required valid port value"), - ) - }) - .collect() - }) - .map(|a| (config.udp, a)), - ); rocket::build() .attach(Template::fairing()) .configure(rocket::Config { @@ -344,7 +364,15 @@ fn rocket() -> _ { rocket::Config::release_default() } }) - .manage(scrape) + .manage( + Scrape::new( + config.scrape, + config.scrape_timeout, + config.scrape_proxy.as_ref(), + config.scrape_proxy_i2p.as_ref(), + ) + .unwrap(), + ) .manage(Storage::init(&config.public, config.list_limit, config.capacity).unwrap()) .manage(Meta { canonical: config.canonical_url, diff --git a/crates/http/src/scrape.rs b/crates/http/src/scrape.rs deleted file mode 100644 index be2e708..0000000 --- a/crates/http/src/scrape.rs +++ /dev/null @@ -1,15 +0,0 @@ -#[derive(rocket::serde::Serialize, Default)] -#[serde(crate = "rocket::serde")] -pub struct Result { - pub leechers: u32, - pub peers: u32, - pub seeders: u32, -} - -pub fn get(scrape: &super::Scrape, id: [u8; 20]) -> Option { - scrape.get(id).map(|s| Result { - leechers: s.leechers, - peers: s.peers, - seeders: s.seeders, - }) -} diff --git a/crates/scrape/Cargo.toml b/crates/scrape/Cargo.toml index 3d09fff..62d42c2 100644 --- a/crates/scrape/Cargo.toml +++ b/crates/scrape/Cargo.toml @@ -1,13 +1,15 @@ [package] name = "btracker-scrape" -version = "0.1.0" +version = "0.2.0" edition = "2024" license = "MIT" readme = "README.md" description = "Shared BitTorrent scrape API for the βtracker project components" -keywords = ["btracker", "bittorrent", "scrape", "udp", "tcp"] +keywords = ["btracker", "bittorrent", "scrape", "tcp"] categories = ["network-programming"] repository = "https://github.com/YGGverse/btracker" [dependencies] -rand = "0.9.2" +anyhow = "1.0.95" +btpeer = "0.11.1" +url = "2.5.7" \ No newline at end of file diff --git a/crates/scrape/LICENSE b/crates/scrape/LICENSE index f13fb1f..4b259c9 100644 --- a/crates/scrape/LICENSE +++ b/crates/scrape/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/crates/scrape/README.md b/crates/scrape/README.md index 1cad069..87d90cb 100644 --- a/crates/scrape/README.md +++ b/crates/scrape/README.md @@ -9,21 +9,4 @@ Shared BitTorrent scrape API for the βtracker project components ``` bash cargo add btracker-scrape -``` - -## Usage - -``` rust -let udp_server = vec![ - SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), - SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, 0, 0, 0)) -]; - -let udp_trackers = vec![ - SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 6969)) -]; - -let scrape = Scrape::init(Some(udp_server, udp_trackers)); - -println!("{:?}", scrape.get([u8; 20])); // hash v1 ``` \ No newline at end of file diff --git a/crates/scrape/src/lib.rs b/crates/scrape/src/lib.rs index e78ffe8..70653b8 100644 --- a/crates/scrape/src/lib.rs +++ b/crates/scrape/src/lib.rs @@ -1,36 +1,88 @@ -mod udp; +use anyhow::{Result as R, bail}; +use btpeer::http::response::scrape::Total; +use std::time::Duration; +use url::Url; -use std::net::SocketAddr; -use udp::Udp; +pub type Result = Total; -#[derive(Default)] -pub struct Result { - pub leechers: u32, - pub peers: u32, - pub seeders: u32, +struct Scrape { + proxy: Option, + timeout: Duration, + tracker: Url, } -pub struct Scrape { - udp: Option, - // tcp: @TODO +impl Scrape { + pub fn new( + tracker: Url, + timeout: u64, + proxy: Option, + proxy_i2p: Option, + ) -> R { + if !tracker.scheme().starts_with("http") { + bail!("HTTP trackers only!") + } + Ok(Self { + proxy: if tracker + .host_str() + .expect("Host is required") + .ends_with(".i2p") + { + if proxy_i2p.is_none() { + bail!("I2P proxy is required for tracker `{tracker}`") + } + proxy_i2p + } else { + proxy + }, + timeout: Duration::from_secs(timeout), + tracker, + }) + } + + pub async fn get(&self, id20: &[[u8; 20]]) -> R { + Ok(btpeer::http::scrape( + &btpeer::http::query::Scrape::new(self.tracker.as_str(), Some(id20))?, + self.timeout, + self.proxy.as_deref(), + ) + .await? + .total) + } } -impl Scrape { - pub fn init(udp: Option<(Vec, Vec)>) -> Self { - Self { - udp: udp.map(|(local, remote)| Udp::init(local, remote)), +pub struct Buffer(Vec); + +impl Buffer { + pub fn new( + trackers: Vec, + timeout: u64, + proxy: Option<&Url>, + proxy_i2p: Option<&Url>, + ) -> R { + let mut this = Vec::with_capacity(trackers.len()); + + for url in trackers { + this.push(Scrape::new( + url, + timeout, + proxy.as_ref().map(|p| p.to_string()), + proxy_i2p.as_ref().map(|p| p.to_string()), + )?) } + + Ok(Self(this)) } - pub fn get(&self, info_hash: [u8; 20]) -> Option { - self.udp.as_ref()?; - let mut t = Result::default(); - if let Some(ref u) = self.udp { - let r = u.scrape(info_hash).ok()?; // @TODO handle - t.leechers += r.leechers; - t.peers += r.peers; - t.seeders += r.seeders; + pub async fn get(&self, id20: &[[u8; 20]]) -> R { + let mut total = Total::default(); + + for this in self.0.iter() { + let result = this.get(id20).await?; + total.complete += result.complete; + total.downloaded += result.downloaded; + total.incomplete += result.incomplete; } - Some(t) + + Ok(total) } } diff --git a/crates/scrape/src/udp.rs b/crates/scrape/src/udp.rs deleted file mode 100644 index 8ee6963..0000000 --- a/crates/scrape/src/udp.rs +++ /dev/null @@ -1,94 +0,0 @@ -use rand::Rng; -use std::{ - io::Error, - net::{SocketAddr, UdpSocket}, - time::Duration, -}; - -struct Route { - socket: UdpSocket, - remote: Vec, -} - -pub struct Udp(Vec); - -impl Udp { - pub fn init(local: Vec, remote: Vec) -> Self { - Self( - local - .into_iter() - .map(|l| { - let socket = UdpSocket::bind(l).unwrap(); - socket - .set_read_timeout(Some(Duration::from_secs(3))) - .unwrap(); - Route { - socket, - remote: if l.is_ipv4() { - remote.iter().filter(|r| r.is_ipv4()).cloned().collect() - } else { - remote.iter().filter(|r| r.is_ipv6()).cloned().collect() - }, - } - }) - .collect(), - ) - } - - pub fn scrape(&self, info_hash: [u8; 20]) -> Result { - let mut t = super::Result::default(); - for route in &self.0 { - for remote in &route.remote { - route.socket.send_to(&connection_request(), remote)?; - - let mut b = [0u8; 16]; - if route.socket.recv(&mut b)? < 16 { - todo!() - } - route.socket.send_to( - &scrape_request( - u64::from_be_bytes(b[8..16].try_into().unwrap()), - rand::rng().random::(), - &[info_hash], - ), - remote, - )?; - - let mut b = [0u8; 1024]; - let l = route.socket.recv(&mut b)?; - if l < 20 { - todo!() - } - - t.seeders += u32::from_be_bytes(b[8..12].try_into().unwrap()); - t.leechers += u32::from_be_bytes(b[12..16].try_into().unwrap()); - t.peers += u32::from_be_bytes(b[16..20].try_into().unwrap()); - } - } - Ok(t) - } -} - -fn connection_request() -> Vec { - let mut b = Vec::new(); - b.extend_from_slice(&0x41727101980u64.to_be_bytes()); - b.extend_from_slice(&0u32.to_be_bytes()); - b.extend_from_slice(&rand::rng().random::().to_be_bytes()); - b -} - -fn scrape_request(connection_id: u64, transaction_id: u32, info_hashes: &[[u8; 20]]) -> Vec { - let mut b = Vec::new(); - b.extend_from_slice(&connection_id.to_be_bytes()); - b.extend_from_slice(&2u32.to_be_bytes()); - b.extend_from_slice(&transaction_id.to_be_bytes()); - // * up to about 74 torrents can be scraped at once - // https://www.bittorrent.org/beps/bep_0015.html - if info_hashes.len() > 74 { - todo!() - } - for hash in info_hashes { - b.extend_from_slice(hash); - } - b -}