From f6cd28224d42332c7151e0256497e27b89369363 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Tue, 5 Apr 2022 23:31:50 +0200 Subject: [PATCH 01/53] wip: support for multiple db drivers --- Cargo.lock | 980 +++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 + src/database.rs | 217 ++------- src/lib.rs | 3 +- src/sqlite_database.rs | 186 ++++++++ src/tracker.rs | 29 +- 6 files changed, 1202 insertions(+), 215 deletions(-) create mode 100644 src/sqlite_database.rs diff --git a/Cargo.lock b/Cargo.lock index 4279ad5b9..bf0fed8f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,23 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom 0.2.3", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -11,6 +28,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "aquatic_udp_protocol" version = "0.1.0" @@ -26,30 +52,111 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "async-trait" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + [[package]] name = "base64" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bigdecimal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256" +dependencies = [ + "num-bigint 0.3.3", + "num-integer", + "num-traits 0.2.14", + "serde 1.0.120", +] + [[package]] name = "binascii" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" +[[package]] +name = "bindgen" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f8523b410d7187a43085e7e064416ea32ded16bd0a4e6fc025e21616d01258f" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "clap", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "which", +] + [[package]] name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitvec" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5237f00a8c86130a0cc317830e558b966dd7850d48a953d998c813f01a41b527" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -90,6 +197,12 @@ dependencies = [ "safemem", ] +[[package]] +name = "bufstream" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" + [[package]] name = "bumpalo" version = "3.8.0" @@ -116,9 +229,18 @@ checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "cc" -version = "1.0.66" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cexpr" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +dependencies = [ + "nom", +] [[package]] name = "cfg-if" @@ -141,10 +263,46 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.14", - "time", + "serde 1.0.120", + "time 0.1.44", "winapi", ] +[[package]] +name = "clang-sys" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cmake" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +dependencies = [ + "cc", +] + [[package]] name = "config" version = "0.11.0" @@ -161,12 +319,34 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "const_fn" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" + [[package]] name = "convert_case" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.1" @@ -176,6 +356,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -185,7 +374,18 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", + "syn", +] + +[[package]] +name = "derive_utils" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532b4c15dccee12c7044f1fcad956e98410860b22231e44a3b827464797ca7bf" +dependencies = [ + "proc-macro2", + "quote", "syn", ] @@ -207,12 +407,31 @@ dependencies = [ "generic-array 0.14.4", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -240,12 +459,40 @@ dependencies = [ "log", ] +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "libz-sys", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.0" @@ -256,6 +503,76 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frunk" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cd67cf7d54b7e72d0ea76f3985c3747d74aee43e0218ad993b7903ba7a5395e" +dependencies = [ + "frunk_core", + "frunk_derives", + "frunk_proc_macros", +] + +[[package]] +name = "frunk_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1246cf43ec80bf8b2505b5c360b8fb999c97dabd17dbb604d85558d5cbc25482" + +[[package]] +name = "frunk_derives" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc4f084ec5a3f031d24ccedeb87ab2c3189a2f33b8d070889073837d5ea09e" +dependencies = [ + "frunk_proc_macro_helpers", + "quote", + "syn", +] + +[[package]] +name = "frunk_proc_macro_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f11257f106c6753f5ffcb8e601fb39c390a088017aaa55b70c526bff15f63e" +dependencies = [ + "frunk_core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "frunk_proc_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a078bd8459eccbb85e0b007b8f756585762a72a9efc53f359b371c3b6351dbcc" +dependencies = [ + "frunk_core", + "frunk_proc_macros_impl", + "proc-macro-hack", +] + +[[package]] +name = "frunk_proc_macros_impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ffba99f0fa4f57e42f57388fbb9a0ca863bc2b4261f3c5570fed579d5df6c32" +dependencies = [ + "frunk_core", + "frunk_proc_macro_helpers", + "proc-macro-hack", + "quote", + "syn", +] + +[[package]] +name = "funty" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847abb9cb65d566acd5942e94aea9c8f547ad02c98e1649326fc0e8910b8b1e" + [[package]] name = "futures" version = "0.3.21" @@ -386,6 +703,12 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" version = "0.3.4" @@ -411,6 +734,15 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + [[package]] name = "headers" version = "0.3.3" @@ -424,7 +756,7 @@ dependencies = [ "http", "mime", "sha-1 0.8.2", - "time", + "time 0.1.44", ] [[package]] @@ -484,6 +816,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.2" @@ -501,7 +839,7 @@ dependencies = [ "httpdate", "itoa", "pin-project", - "socket2", + "socket2 0.3.19", "tokio", "tower-service", "tracing", @@ -526,7 +864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -547,6 +885,18 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "io-enum" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e3306b0f260aad2872563eb0d5d1a59f2420fad270a661dce59a01e92d806b" +dependencies = [ + "autocfg", + "derive_utils", + "quote", + "syn", +] + [[package]] name = "itoa" version = "0.4.7" @@ -568,13 +918,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "lexical" +version = "5.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f404a90a744e32e8be729034fc33b90cf2a56418fbf594d69aa3c0214ad414e5" +dependencies = [ + "cfg-if 1.0.0", + "lexical-core", +] + [[package]] name = "lexical-core" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ - "arrayvec", + "arrayvec 0.5.2", "bitflags", "cfg-if 1.0.0", "ryu", @@ -583,9 +949,19 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.101" +version = "0.2.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09" + +[[package]] +name = "libloading" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if 1.0.0", + "winapi", +] [[package]] name = "libsqlite3-sys" @@ -597,6 +973,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-sys" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -621,6 +1008,15 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "lru" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea2d928b485416e8908cff2d97d621db22b27f7b3b6729e438bcf42c671ba91" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -642,6 +1038,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -658,6 +1063,16 @@ dependencies = [ "unicase", ] +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.7" @@ -677,7 +1092,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ - "socket2", + "socket2 0.3.19", "winapi", ] @@ -699,6 +1114,109 @@ dependencies = [ "twoway", ] +[[package]] +name = "mysql" +version = "21.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f5abe1c0f91831afd4d35298c08d958e80144869757b913891e5b0d00c2c96" +dependencies = [ + "bufstream", + "bytes", + "io-enum", + "libc", + "lru", + "mysql_common", + "named_pipe", + "native-tls", + "nix", + "once_cell", + "pem", + "percent-encoding", + "serde 1.0.120", + "serde_json", + "socket2 0.4.4", + "twox-hash", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fa08ec695a40ed899b1239e81d0d74de5b40802d4fc8b513e2c541717c434e" +dependencies = [ + "base64", + "bigdecimal", + "bindgen", + "bitflags", + "bitvec", + "byteorder", + "bytes", + "cc", + "chrono", + "cmake", + "crc32fast", + "flate2", + "frunk", + "lazy_static", + "lexical", + "num-bigint 0.4.3", + "num-traits 0.2.14", + "rand 0.8.4", + "regex", + "rust_decimal", + "saturating", + "serde 1.0.120", + "serde_json", + "sha1", + "sha2", + "smallvec", + "subprocess", + "thiserror", + "time 0.2.27", + "uuid", +] + +[[package]] +name = "named_pipe" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset", +] + [[package]] name = "nom" version = "5.1.2" @@ -719,6 +1237,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -758,22 +1298,55 @@ dependencies = [ ] [[package]] -name = "once_cell" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" - -[[package]] -name = "opaque-debug" -version = "0.2.3" +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] -name = "opaque-debug" -version = "0.3.0" +name = "openssl-sys" +version = "0.9.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] [[package]] name = "parking_lot" @@ -800,6 +1373,23 @@ dependencies = [ "winapi", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "pem" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" +dependencies = [ + "base64", + "once_cell", + "regex", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -850,6 +1440,12 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.24" @@ -885,6 +1481,16 @@ dependencies = [ "scheduled-thread-pool", ] +[[package]] +name = "r2d2_mysql" +version = "21.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d05145690b395f5515feff202b8f4b9429c500f423ef7129175155c3c3a9e2" +dependencies = [ + "mysql", + "r2d2", +] + [[package]] name = "r2d2_sqlite" version = "0.16.0" @@ -895,6 +1501,12 @@ dependencies = [ "rusqlite", ] +[[package]] +name = "radium" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" + [[package]] name = "rand" version = "0.7.3" @@ -1039,7 +1651,7 @@ dependencies = [ "lru-cache", "memchr", "smallvec", - "time", + "time 0.1.44", ] [[package]] @@ -1048,13 +1660,39 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" +[[package]] +name = "rust_decimal" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37baa70cf8662d2ba1c1868c5983dda16ef32b105cce41fb5c47e72936a90b3" +dependencies = [ + "arrayvec 0.7.2", + "num-traits 0.2.14", + "serde 1.0.120", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.4", ] [[package]] @@ -1082,6 +1720,22 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "scheduled-thread-pool" version = "0.2.5" @@ -1113,12 +1767,50 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "0.8.23" @@ -1224,6 +1916,40 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "signal-hook-registry" version = "1.3.0" @@ -1256,18 +1982,102 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version 0.2.3", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde 1.0.120", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde 1.0.120", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "subprocess" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "syn" version = "1.0.67" @@ -1279,6 +2089,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.2.0" @@ -1293,6 +2109,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.26" @@ -1324,6 +2158,44 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + [[package]] name = "tinyvec" version = "1.3.1" @@ -1432,6 +2304,7 @@ name = "torrust-tracker" version = "2.2.0" dependencies = [ "aquatic_udp_protocol", + "async-trait", "binascii", "byteorder", "chrono", @@ -1443,6 +2316,7 @@ dependencies = [ "log", "percent-encoding", "r2d2", + "r2d2_mysql", "r2d2_sqlite", "rand 0.8.4", "serde 1.0.120", @@ -1516,6 +2390,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "twox-hash" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" +dependencies = [ + "cfg-if 1.0.0", + "rand 0.8.4", + "static_assertions", +] + [[package]] name = "typenum" version = "1.12.0" @@ -1546,6 +2431,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -1576,12 +2467,24 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.2" @@ -1714,6 +2617,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1730,12 +2642,30 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "wyz" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129e027ad65ce1453680623c3fb5163cbf7107bfe1aa32257e7d0e63f9ced188" +dependencies = [ + "tap", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 084a7cfb5..b29bf09f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ log = {version = "0.4", features = ["release_max_level_info"]} fern = "0.6" chrono = "0.4" byteorder = "1" +r2d2_mysql = "21.0.0" r2d2_sqlite = "0.16.0" r2d2 = "0.8.8" rand = "0.8.4" @@ -32,3 +33,4 @@ derive_more = "0.99" thiserror = "1.0" aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } futures = "0.3.21" +async-trait = "0.1.52" diff --git a/src/database.rs b/src/database.rs index 82d26d6d8..63d06c4f9 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,196 +1,63 @@ use std::collections::BTreeMap; -use crate::{InfoHash, AUTH_KEY_LENGTH, TorrentEntry}; -use log::debug; -use r2d2_sqlite::{SqliteConnectionManager, rusqlite}; -use r2d2::{Pool}; -use r2d2_sqlite::rusqlite::NO_PARAMS; +use crate::{InfoHash, TorrentEntry}; use crate::key_manager::AuthKey; -use std::str::FromStr; +use crate::sqlite_database::SqliteDatabase; +use async_trait::async_trait; +use derive_more::{Display, Error}; -pub struct SqliteDatabase { - pool: Pool +pub enum DatabaseDrivers { + Sqlite3, + MySQL } -impl SqliteDatabase { - pub fn new(db_path: &str) -> Result { - let sqlite_connection_manager = SqliteConnectionManager::file(db_path); - let sqlite_pool = r2d2::Pool::new(sqlite_connection_manager).expect("Failed to create r2d2 SQLite connection pool."); - let sqlite_database = SqliteDatabase { - pool: sqlite_pool - }; - - if let Err(error) = SqliteDatabase::create_database_tables(&sqlite_database.pool) { - return Err(error) - }; - - Ok(sqlite_database) - } - - pub fn create_database_tables(pool: &Pool) -> Result { - let create_whitelist_table = " - CREATE TABLE IF NOT EXISTS whitelist ( - id integer PRIMARY KEY AUTOINCREMENT, - info_hash VARCHAR(20) NOT NULL UNIQUE - );".to_string(); - - let create_torrents_table = " - CREATE TABLE IF NOT EXISTS torrents ( - id integer PRIMARY KEY AUTOINCREMENT, - info_hash VARCHAR(20) NOT NULL UNIQUE, - completed INTEGER DEFAULT 0 NOT NULL - );".to_string(); - - let create_keys_table = format!(" - CREATE TABLE IF NOT EXISTS keys ( - id integer PRIMARY KEY AUTOINCREMENT, - key VARCHAR({}) NOT NULL UNIQUE, - valid_until INT(10) NOT NULL - );", AUTH_KEY_LENGTH as i8); - - let conn = pool.get().unwrap(); - match conn.execute(&create_whitelist_table, NO_PARAMS) { - Ok(updated) => { - match conn.execute(&create_keys_table, NO_PARAMS) { - Ok(updated2) => { - match conn.execute(&create_torrents_table, NO_PARAMS) { - Ok(updated3) => Ok(updated + updated2 + updated3), - Err(e) => { - debug!("{:?}", e); - Err(e) - } - } - } - Err(e) => { - debug!("{:?}", e); - Err(e) - } - } - } - Err(e) => { - debug!("{:?}", e); - Err(e) - } +pub fn connect_database(db_driver: &DatabaseDrivers, db_path: &str) -> Result { + match db_driver { + DatabaseDrivers::Sqlite3 => { + let db = SqliteDatabase::new(db_path)?; + Ok(db) } - } - - pub async fn load_persistent_torrent_data(&self) -> Result, rusqlite::Error> { - let conn = self.pool.get().unwrap(); - let mut stmt = conn.prepare("SELECT info_hash, completed FROM torrents")?; - - let torrent_iter = stmt.query_map(NO_PARAMS, |row| { - let info_hash_string: String = row.get(0)?; - let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); - let completed: u32 = row.get(1)?; - Ok((info_hash, completed)) - })?; - - let torrents: Vec<(InfoHash, u32)> = torrent_iter.filter_map(|x| x.ok() ).collect(); - - Ok(torrents) - } - - pub async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), rusqlite::Error> { - let mut conn = self.pool.get().unwrap(); - let db_transaction = conn.transaction()?; - - for (info_hash, torrent_entry) in torrents { - let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + _ => { + let db = SqliteDatabase::new(db_path)?; + Ok(db) } - - let _ = db_transaction.commit(); - - Ok(()) } +} - pub async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { - let conn = self.pool.get().unwrap(); - let mut stmt = conn.prepare("SELECT info_hash FROM whitelist WHERE info_hash = ?")?; - let mut rows = stmt.query(&[info_hash])?; +#[async_trait] +pub trait Database: Sync + Send { + fn create_database_tables(&self) -> Result; - if let Some(row) = rows.next()? { - let info_hash: String = row.get(0).unwrap(); + async fn load_persistent_torrent_data(&self) -> Result, Error>; - // should never be able to fail - Ok(InfoHash::from_str(&info_hash).unwrap()) - } else { - Err(rusqlite::Error::QueryReturnedNoRows) - } - } + async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), Error>; - pub async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { - let conn = self.pool.get().unwrap(); - match conn.execute("INSERT INTO whitelist (info_hash) VALUES (?)", &[info_hash.to_string()]) { - Ok(updated) => { - if updated > 0 { return Ok(updated) } - Err(rusqlite::Error::ExecuteReturnedResults) - }, - Err(e) => { - debug!("{:?}", e); - Err(e) - } - } - } + async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result; - pub async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { - let conn = self.pool.get().unwrap(); - match conn.execute("DELETE FROM whitelist WHERE info_hash = ?", &[info_hash.to_string()]) { - Ok(updated) => { - if updated > 0 { return Ok(updated) } - Err(rusqlite::Error::ExecuteReturnedResults) - }, - Err(e) => { - debug!("{:?}", e); - Err(e) - } - } - } + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result; - pub async fn get_key_from_keys(&self, key: &str) -> Result { - let conn = self.pool.get().unwrap(); - let mut stmt = conn.prepare("SELECT key, valid_until FROM keys WHERE key = ?")?; - let mut rows = stmt.query(&[key.to_string()])?; + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result; - if let Some(row) = rows.next()? { - let key: String = row.get(0).unwrap(); - let valid_until_i64: i64 = row.get(1).unwrap(); + async fn get_key_from_keys(&self, key: &str) -> Result; - Ok(AuthKey { - key, - valid_until: Some(valid_until_i64 as u64) - }) - } else { - Err(rusqlite::Error::QueryReturnedNoRows) - } - } + async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result; - pub async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result { - let conn = self.pool.get().unwrap(); - match conn.execute("INSERT INTO keys (key, valid_until) VALUES (?1, ?2)", - &[auth_key.key.to_string(), auth_key.valid_until.unwrap().to_string()] - ) { - Ok(updated) => { - if updated > 0 { return Ok(updated) } - Err(rusqlite::Error::ExecuteReturnedResults) - }, - Err(e) => { - debug!("{:?}", e); - Err(e) - } - } - } + async fn remove_key_from_keys(&self, key: String) -> Result; +} + +#[derive(Debug, Display, PartialEq, Error)] +#[allow(dead_code)] +pub enum Error { + #[display(fmt = "Query returned no rows.")] + QueryReturnedNoRows, + #[display(fmt = "Invalid query.")] + InvalidQuery, +} - pub async fn remove_key_from_keys(&self, key: String) -> Result { - let conn = self.pool.get().unwrap(); - match conn.execute("DELETE FROM keys WHERE key = ?", &[key]) { - Ok(updated) => { - if updated > 0 { return Ok(updated) } - Err(rusqlite::Error::ExecuteReturnedResults) - }, - Err(e) => { - debug!("{:?}", e); - Err(e) - } +impl From for Error { + fn from(e: r2d2_sqlite::rusqlite::Error) -> Self { + match e { + r2d2_sqlite::rusqlite::Error::QueryReturnedNoRows => Error::QueryReturnedNoRows, + _ => Error::InvalidQuery } } } diff --git a/src/lib.rs b/src/lib.rs index c055cfae4..a9692ac66 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,11 +3,12 @@ pub mod tracker; pub mod http_api_server; pub mod common; pub mod utils; -pub mod database; +pub mod sqlite_database; pub mod key_manager; pub mod logging; pub mod torrust_udp_tracker; pub mod torrust_http_tracker; +pub mod database; pub use self::config::*; pub use torrust_udp_tracker::server::*; diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs new file mode 100644 index 000000000..5bd081c6f --- /dev/null +++ b/src/sqlite_database.rs @@ -0,0 +1,186 @@ +use std::collections::BTreeMap; +use crate::{InfoHash, AUTH_KEY_LENGTH, TorrentEntry, database}; +use log::debug; +use r2d2_sqlite::{SqliteConnectionManager}; +use r2d2::{Pool}; +use r2d2_sqlite::rusqlite::NO_PARAMS; +use crate::key_manager::AuthKey; +use std::str::FromStr; +use crate::database::Database; +use async_trait::async_trait; + +pub struct SqliteDatabase { + pool: Pool +} + +impl SqliteDatabase { + pub fn new(db_path: &str) -> Result { + let cm = SqliteConnectionManager::file(db_path); + let pool = Pool::new(cm).expect("Failed to create r2d2 SQLite connection pool."); + Ok(SqliteDatabase { + pool + }) + } +} + +#[async_trait] +impl Database for SqliteDatabase { + fn create_database_tables(&self) -> Result { + let create_whitelist_table = " + CREATE TABLE IF NOT EXISTS whitelist ( + id integer PRIMARY KEY AUTOINCREMENT, + info_hash VARCHAR(20) NOT NULL UNIQUE + );".to_string(); + + let create_torrents_table = " + CREATE TABLE IF NOT EXISTS torrents ( + id integer PRIMARY KEY AUTOINCREMENT, + info_hash VARCHAR(20) NOT NULL UNIQUE, + completed INTEGER DEFAULT 0 NOT NULL + );".to_string(); + + let create_keys_table = format!(" + CREATE TABLE IF NOT EXISTS keys ( + id integer PRIMARY KEY AUTOINCREMENT, + key VARCHAR({}) NOT NULL UNIQUE, + valid_until INT(10) NOT NULL + );", AUTH_KEY_LENGTH as i8); + + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + conn.execute(&create_whitelist_table, NO_PARAMS) + .and_then(|_| conn.execute(&create_whitelist_table, NO_PARAMS)) + .and_then(|_| conn.execute(&create_keys_table, NO_PARAMS)) + .and_then(|_| conn.execute(&create_torrents_table, NO_PARAMS)) + .map_err(|_| database::Error::InvalidQuery) + } + + async fn load_persistent_torrent_data(&self) -> Result, database::Error> { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let mut stmt = conn.prepare("SELECT info_hash, completed FROM torrents")?; + + let torrent_iter = stmt.query_map(NO_PARAMS, |row| { + let info_hash_string: String = row.get(0)?; + let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); + let completed: u32 = row.get(1)?; + Ok((info_hash, completed)) + })?; + + let torrents: Vec<(InfoHash, u32)> = torrent_iter.filter_map(|x| x.ok() ).collect(); + + Ok(torrents) + } + + async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), database::Error> { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let db_transaction = conn.transaction()?; + + for (info_hash, torrent_entry) in torrents { + let (_seeders, completed, _leechers) = torrent_entry.get_stats(); + let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + } + + let _ = db_transaction.commit(); + + Ok(()) + } + + async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let mut stmt = conn.prepare("SELECT info_hash FROM whitelist WHERE info_hash = ?")?; + let mut rows = stmt.query(&[info_hash])?; + + if let Some(row) = rows.next()? { + let info_hash: String = row.get(0).unwrap(); + + // should never be able to fail + Ok(InfoHash::from_str(&info_hash).unwrap()) + } else { + Err(database::Error::InvalidQuery) + } + } + + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.execute("INSERT INTO whitelist (info_hash) VALUES (?)", &[info_hash.to_string()]) { + Ok(updated) => { + if updated > 0 { return Ok(updated) } + Err(database::Error::QueryReturnedNoRows) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } + + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.execute("DELETE FROM whitelist WHERE info_hash = ?", &[info_hash.to_string()]) { + Ok(updated) => { + if updated > 0 { return Ok(updated) } + Err(database::Error::QueryReturnedNoRows) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } + + async fn get_key_from_keys(&self, key: &str) -> Result { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let mut stmt = conn.prepare("SELECT key, valid_until FROM keys WHERE key = ?")?; + let mut rows = stmt.query(&[key.to_string()])?; + + if let Some(row) = rows.next()? { + let key: String = row.get(0).unwrap(); + let valid_until_i64: i64 = row.get(1).unwrap(); + + Ok(AuthKey { + key, + valid_until: Some(valid_until_i64 as u64) + }) + } else { + Err(database::Error::QueryReturnedNoRows) + } + } + + async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.execute("INSERT INTO keys (key, valid_until) VALUES (?1, ?2)", + &[auth_key.key.to_string(), auth_key.valid_until.unwrap().to_string()] + ) { + Ok(updated) => { + if updated > 0 { return Ok(updated) } + Err(database::Error::QueryReturnedNoRows) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } + + async fn remove_key_from_keys(&self, key: String) -> Result { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.execute("DELETE FROM keys WHERE key = ?", &[key]) { + Ok(updated) => { + if updated > 0 { return Ok(updated) } + Err(database::Error::QueryReturnedNoRows) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } +} diff --git a/src/tracker.rs b/src/tracker.rs index 98c5be730..24bd645d1 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -5,14 +5,14 @@ use std::collections::BTreeMap; use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use crate::common::{AnnounceEventDef, InfoHash, NumberOfBytesDef, PeerId}; use std::net::{IpAddr, SocketAddr}; -use crate::{Configuration, key_manager, MAX_SCRAPE_TORRENTS}; +use crate::{Configuration, database, key_manager, MAX_SCRAPE_TORRENTS}; use std::collections::btree_map::Entry; -use crate::database::SqliteDatabase; use std::sync::Arc; use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; use log::{debug}; use crate::key_manager::AuthKey; -use r2d2_sqlite::rusqlite; +use crate::database::{Database, DatabaseDrivers}; +use crate::key_manager::Error::KeyInvalid; use crate::torrust_http_tracker::AnnounceRequest; #[derive(Serialize, Deserialize, Clone, PartialEq)] @@ -265,18 +265,19 @@ pub struct TrackerStats { pub struct TorrentTracker { pub config: Arc, torrents: tokio::sync::RwLock>, - database: SqliteDatabase, + database: Box, stats: tokio::sync::RwLock, } impl TorrentTracker { - pub fn new(config: Arc) -> Result { - let database = SqliteDatabase::new(&config.db_path)?; + pub fn new(config: Arc) -> Result { + let db_driver = DatabaseDrivers::Sqlite3; + let database = database::connect_database(&db_driver, "data")?; Ok(TorrentTracker { config, torrents: RwLock::new(std::collections::BTreeMap::new()), - database, + database: Box::new(database), stats: RwLock::new(TrackerStats { tcp4_connections_handled: 0, tcp4_announces_handled: 0, @@ -306,7 +307,7 @@ impl TorrentTracker { self.config.mode == TrackerMode::ListedMode || self.config.mode == TrackerMode::PrivateListedMode } - pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { + pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { let auth_key = key_manager::generate_auth_key(seconds_valid); // add key to database @@ -315,12 +316,12 @@ impl TorrentTracker { Ok(auth_key) } - pub async fn remove_auth_key(&self, key: String) -> Result { + pub async fn remove_auth_key(&self, key: String) -> Result { self.database.remove_key_from_keys(key).await } pub async fn verify_auth_key(&self, auth_key: &AuthKey) -> Result<(), key_manager::Error> { - let db_key = self.database.get_key_from_keys(&auth_key.key).await?; + let db_key = self.database.get_key_from_keys(&auth_key.key).await.map_err(|_| KeyInvalid)?; key_manager::verify_auth_key(&db_key) } @@ -353,7 +354,7 @@ impl TorrentTracker { } // Loading the torrents into memory - pub async fn load_torrents(&self) -> Result<(), rusqlite::Error> { + pub async fn load_torrents(&self) -> Result<(), database::Error> { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { @@ -364,18 +365,18 @@ impl TorrentTracker { } // Saving the torrents from memory - pub async fn save_torrents(&self) -> Result<(), rusqlite::Error> { + pub async fn save_torrents(&self) -> Result<(), database::Error> { let torrents = self.torrents.read().await; self.database.save_persistent_torrent_data(&*torrents).await } // Adding torrents is not relevant to public trackers. - pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result { + pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result { self.database.add_info_hash_to_whitelist(info_hash.clone()).await } // Removing torrents is not relevant to public trackers. - pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result { + pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result { self.database.remove_info_hash_from_whitelist(info_hash.clone()).await } From a9956381480bba15e00df08cab43b2e807e6a79f Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 10 Apr 2022 09:45:49 +0200 Subject: [PATCH 02/53] Fixing torrent cleanup, adding defragmentation of memory, and fixed response failure and bumping the version --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/torrust_http_tracker/response.rs | 1 + src/tracker.rs | 76 +++++++++++++++------------- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf0fed8f5..68f52fa18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2301,7 +2301,7 @@ dependencies = [ [[package]] name = "torrust-tracker" -version = "2.2.0" +version = "2.2.1" dependencies = [ "aquatic_udp_protocol", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b29bf09f5..e3e57c378 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "torrust-tracker" -version = "2.2.0" +version = "2.2.1" license = "AGPL-3.0" authors = ["Mick van Dijke "] description = "A feature rich BitTorrent tracker." diff --git a/src/torrust_http_tracker/response.rs b/src/torrust_http_tracker/response.rs index d459a6fdd..af27bc5e9 100644 --- a/src/torrust_http_tracker/response.rs +++ b/src/torrust_http_tracker/response.rs @@ -86,6 +86,7 @@ impl ScrapeResponse { #[derive(Serialize)] pub struct ErrorResponse { + #[serde(rename = "failure reason")] pub failure_reason: String } diff --git a/src/tracker.rs b/src/tracker.rs index 24bd645d1..b79ffb6cf 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -9,7 +9,7 @@ use crate::{Configuration, database, key_manager, MAX_SCRAPE_TORRENTS}; use std::collections::btree_map::Entry; use std::sync::Arc; use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; -use log::{debug}; +use log::info; use crate::key_manager::AuthKey; use crate::database::{Database, DatabaseDrivers}; use crate::key_manager::Error::KeyInvalid; @@ -456,47 +456,53 @@ impl TorrentTracker { self.stats.read().await } - // remove torrents without peers + // remove torrents without peers if enabled, and defragment memory pub async fn cleanup_torrents(&self) { - debug!("Cleaning torrents.."); - let mut lock = self.torrents.write().await; - let db: &mut BTreeMap = &mut *lock; - let mut torrents_to_remove = Vec::new(); - - for (k, torrent_entry) in db.iter_mut() { - // timed-out peers.. - { - let mut peers_to_remove = Vec::new(); - let torrent_peers = &mut torrent_entry.peers; - - for (peer_id, peer) in torrent_peers.iter() { - if peer.is_seeder() { - if peer.updated.elapsed() > std::time::Duration::from_secs(self.config.peer_timeout as u64) { - // remove seeders after 5 minutes since last update... - peers_to_remove.push(peer_id.clone()); - torrent_entry.seeders -= 1; - } - } else if peer.updated.elapsed() > std::time::Duration::from_secs(self.config.peer_timeout as u64) { - // remove peers after 2 hours since last update... - peers_to_remove.push(peer_id.clone()); - } - } + info!("Cleaning torrents..."); + let lock = self.torrents.write().await; + + // First we create a mapping of all the torrent hashes in a vector, and we use this to iterate through the btreemap. + // Every hash we have handled, we remove from the btreemap completely, and push it to the top. + let mut torrent_hashes: Vec = Vec::new(); + for (k, _torrent_entry) in lock.iter() { + torrent_hashes.push(k.clone()); + } + + drop(lock); + + // Let's iterate through all torrents, and parse. + for hash in torrent_hashes.iter() { + let mut torrent = TorrentEntry{ + peers: BTreeMap::new(), + completed: 0, + seeders: 0 + }; - for peer_id in peers_to_remove.iter() { - torrent_peers.remove(peer_id); + let lock = self.torrents.write().await; + let torrent_data = lock.get(hash).unwrap().clone(); + drop(lock); + + torrent.completed = torrent_data.completed.clone(); + for (peer_id, peer) in torrent_data.peers.iter() { + if peer.updated.elapsed() > std::time::Duration::from_secs(self.config.peer_timeout as u64) { + continue; + } + torrent.peers.insert(peer_id.clone(), peer.clone()); + if peer.is_seeder() { + torrent.seeders += 1; } } - + let mut lock = self.torrents.write().await; + lock.remove(hash); if self.config.mode.clone() == TrackerMode::PublicMode && self.config.cleanup_peerless && !self.config.persistence { - // peer-less torrents.. - if torrent_entry.peers.len() == 0 { - torrents_to_remove.push(k.clone()); + if torrent.peers.len() != 0 { + lock.insert(hash.clone(), torrent); } + } else { + lock.insert(hash.clone(), torrent); } + drop(lock); } - - for info_hash in torrents_to_remove { - db.remove(&info_hash); - } + info!("Torrents cleaned up."); } } From 5baea30dfe2f9cb2f3005de63775622bf3bb3396 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 11 Apr 2022 01:01:03 +0200 Subject: [PATCH 03/53] feat: added MySQL database support --- src/config.rs | 3 + src/database.rs | 28 +++++-- src/lib.rs | 1 + src/mysql_database.rs | 183 +++++++++++++++++++++++++++++++++++++++++ src/sqlite_database.rs | 4 +- src/tracker.rs | 7 +- 6 files changed, 212 insertions(+), 14 deletions(-) create mode 100644 src/mysql_database.rs diff --git a/src/config.rs b/src/config.rs index 94b37464d..529ae07fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -8,6 +8,7 @@ use std::net::{IpAddr}; use std::path::Path; use std::str::FromStr; use config::{ConfigError, Config, File}; +use crate::database::DatabaseDrivers; #[derive(Serialize, Deserialize, PartialEq)] pub enum TrackerServer { @@ -50,6 +51,7 @@ pub struct HttpApiConfig { pub struct Configuration { pub log_level: Option, pub mode: TrackerMode, + pub db_driver: DatabaseDrivers, pub db_path: String, pub persistence: bool, pub cleanup_interval: Option, @@ -132,6 +134,7 @@ impl Configuration { let mut configuration = Configuration { log_level: Option::from(String::from("info")), mode: TrackerMode::PublicMode, + db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), persistence: false, cleanup_interval: Some(600), diff --git a/src/database.rs b/src/database.rs index 63d06c4f9..0c0e4a303 100644 --- a/src/database.rs +++ b/src/database.rs @@ -4,28 +4,38 @@ use crate::key_manager::AuthKey; use crate::sqlite_database::SqliteDatabase; use async_trait::async_trait; use derive_more::{Display, Error}; +use log::debug; +use crate::mysql_database::MysqlDatabase; +use serde::{Serialize, Deserialize}; +#[derive(Serialize, Deserialize, Debug)] pub enum DatabaseDrivers { Sqlite3, MySQL } -pub fn connect_database(db_driver: &DatabaseDrivers, db_path: &str) -> Result { - match db_driver { +pub fn connect_database(db_driver: &DatabaseDrivers, db_path: &str) -> Result, r2d2::Error> { + debug!("{:?}", db_driver); + + let database: Box = match db_driver { DatabaseDrivers::Sqlite3 => { let db = SqliteDatabase::new(db_path)?; - Ok(db) + Box::new(db) } - _ => { - let db = SqliteDatabase::new(db_path)?; - Ok(db) + DatabaseDrivers::MySQL => { + let db = MysqlDatabase::new(db_path)?; + Box::new(db) } - } + }; + + database.create_database_tables().expect("Could not create database tables."); + + Ok(database) } #[async_trait] pub trait Database: Sync + Send { - fn create_database_tables(&self) -> Result; + fn create_database_tables(&self) -> Result<(), Error>; async fn load_persistent_torrent_data(&self) -> Result, Error>; @@ -51,6 +61,8 @@ pub enum Error { QueryReturnedNoRows, #[display(fmt = "Invalid query.")] InvalidQuery, + #[display(fmt = "Database error.")] + DatabaseError, } impl From for Error { diff --git a/src/lib.rs b/src/lib.rs index a9692ac66..addc7e1a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod logging; pub mod torrust_udp_tracker; pub mod torrust_http_tracker; pub mod database; +pub mod mysql_database; pub use self::config::*; pub use torrust_udp_tracker::server::*; diff --git a/src/mysql_database.rs b/src/mysql_database.rs new file mode 100644 index 000000000..f6907f309 --- /dev/null +++ b/src/mysql_database.rs @@ -0,0 +1,183 @@ +use std::collections::BTreeMap; +use crate::{InfoHash, AUTH_KEY_LENGTH, TorrentEntry, database}; +use log::debug; +use r2d2::{Pool}; +use crate::key_manager::AuthKey; +use std::str::FromStr; +use crate::database::Database; +use async_trait::async_trait; +use r2d2_mysql::mysql::{Opts, OptsBuilder, params, TxOpts}; +use r2d2_mysql::mysql::prelude::Queryable; +use r2d2_mysql::MysqlConnectionManager; + +pub struct MysqlDatabase { + pool: Pool +} + +impl MysqlDatabase { + pub fn new(db_path: &str) -> Result { + let opts = Opts::from_url(&db_path).expect("Failed to connect to MySQL database."); + let builder = OptsBuilder::from_opts(opts); + let manager = MysqlConnectionManager::new(builder); + let pool = r2d2::Pool::builder().build(manager).expect("Failed to create r2d2 MySQL connection pool."); + + Ok(Self { + pool + }) + } +} + +#[async_trait] +impl Database for MysqlDatabase { + fn create_database_tables(&self) -> Result<(), database::Error> { + let create_whitelist_table = " + CREATE TABLE IF NOT EXISTS whitelist ( + id integer PRIMARY KEY AUTO_INCREMENT, + info_hash VARCHAR(20) NOT NULL UNIQUE + );".to_string(); + + let create_torrents_table = " + CREATE TABLE IF NOT EXISTS torrents ( + id integer PRIMARY KEY AUTO_INCREMENT, + info_hash VARCHAR(20) NOT NULL UNIQUE, + completed INTEGER DEFAULT 0 NOT NULL + );".to_string(); + + let create_keys_table = format!(" + CREATE TABLE IF NOT EXISTS `keys` ( + `id` INT NOT NULL AUTO_INCREMENT, + `key` VARCHAR({}) NOT NULL, + `valid_until` INT(10) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE (`key`) + );", AUTH_KEY_LENGTH as i8); + + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + conn.query_drop(&create_torrents_table).expect("Could not create torrents table."); + conn.query_drop(&create_keys_table).expect("Could not create keys table."); + conn.query_drop(&create_whitelist_table).expect("Could not create whitelist table."); + + Ok(()) + } + + async fn load_persistent_torrent_data(&self) -> Result, database::Error> { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT info_hash, completed FROM torrents", |(info_hash_string, completed): (String, u32)| { + let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); + (info_hash, completed) + }).map_err(|_| database::Error::QueryReturnedNoRows)?; + + Ok(torrents) + } + + async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), database::Error> { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; + + for (info_hash, torrent_entry) in torrents { + let (_seeders, completed, _leechers) = torrent_entry.get_stats(); + let _ = db_transaction.exec_drop("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", (info_hash.to_string(), completed.to_string())); + } + + let _ = db_transaction.commit(); + + Ok(()) + } + + async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.exec_first::("SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", params! { info_hash => info_hash }) + .map_err(|_| database::Error::QueryReturnedNoRows)? { + Some(info_hash) => { + Ok(InfoHash::from_str(&info_hash).unwrap()) + }, + None => { + Err(database::Error::InvalidQuery) + } + } + } + + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let info_hash_str = info_hash.to_string(); + + match conn.exec_drop("INSERT INTO whitelist (info_hash) VALUES (:info_hash_str)", params! { info_hash_str }) { + Ok(_) => { + Ok(1) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } + + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let info_hash = info_hash.to_string(); + + match conn.exec_drop("DELETE FROM whitelist WHERE info_hash = :info_hash", params! { info_hash }) { + Ok(_) => { + Ok(1) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } + + async fn get_key_from_keys(&self, key: &str) -> Result { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.exec_first::<(String, i64), _, _>("SELECT `key`, valid_until FROM `keys` WHERE `key` = :key", params! { key => key }) + .map_err(|_| database::Error::QueryReturnedNoRows)? { + Some((key, valid_until)) => { + Ok(AuthKey { + key, + valid_until: Some(valid_until as u64) + }) + }, + None => { + Err(database::Error::InvalidQuery) + } + } + } + + async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + let key = auth_key.key.to_string(); + let valid_until = auth_key.valid_until.unwrap_or(0).to_string(); + + match conn.exec_drop("INSERT INTO `keys` (`key`, valid_until) VALUES (:key, :valid_until)", params! { key, valid_until }) { + Ok(_) => { + Ok(1) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } + + async fn remove_key_from_keys(&self, key: String) -> Result { + let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + + match conn.exec_drop("DELETE FROM `keys` WHERE key = :key", params! { key }) { + Ok(_) => { + Ok(1) + }, + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } + } +} diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index 5bd081c6f..c54a3c79c 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -25,7 +25,7 @@ impl SqliteDatabase { #[async_trait] impl Database for SqliteDatabase { - fn create_database_tables(&self) -> Result { + fn create_database_tables(&self) -> Result<(), database::Error> { let create_whitelist_table = " CREATE TABLE IF NOT EXISTS whitelist ( id integer PRIMARY KEY AUTOINCREMENT, @@ -49,10 +49,10 @@ impl Database for SqliteDatabase { let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; conn.execute(&create_whitelist_table, NO_PARAMS) - .and_then(|_| conn.execute(&create_whitelist_table, NO_PARAMS)) .and_then(|_| conn.execute(&create_keys_table, NO_PARAMS)) .and_then(|_| conn.execute(&create_torrents_table, NO_PARAMS)) .map_err(|_| database::Error::InvalidQuery) + .map(|_| ()) } async fn load_persistent_torrent_data(&self) -> Result, database::Error> { diff --git a/src/tracker.rs b/src/tracker.rs index b79ffb6cf..1caccff23 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; use log::info; use crate::key_manager::AuthKey; -use crate::database::{Database, DatabaseDrivers}; +use crate::database::{Database}; use crate::key_manager::Error::KeyInvalid; use crate::torrust_http_tracker::AnnounceRequest; @@ -271,13 +271,12 @@ pub struct TorrentTracker { impl TorrentTracker { pub fn new(config: Arc) -> Result { - let db_driver = DatabaseDrivers::Sqlite3; - let database = database::connect_database(&db_driver, "data")?; + let database = database::connect_database(&config.db_driver, &config.db_path)?; Ok(TorrentTracker { config, torrents: RwLock::new(std::collections::BTreeMap::new()), - database: Box::new(database), + database, stats: RwLock::new(TrackerStats { tcp4_connections_handled: 0, tcp4_announces_handled: 0, From f407b9e686dd0a713c9901132ad52590d7885669 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 11 Apr 2022 23:45:38 +0200 Subject: [PATCH 04/53] fix: error when running in PrivateMode or PrivateListedMode --- src/config.rs | 27 ++++++++------------------- src/main.rs | 38 ++++++++++++++++++++++---------------- src/tracker.rs | 8 ++++---- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/src/config.rs b/src/config.rs index 529ae07fe..f9166e577 100644 --- a/src/config.rs +++ b/src/config.rs @@ -16,18 +16,17 @@ pub enum TrackerServer { HTTP } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct UdpTrackerConfig { pub enabled: bool, pub bind_address: String, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct HttpTrackerConfig { pub enabled: bool, pub bind_address: String, pub ssl_enabled: bool, - pub ssl_bind_address: String, #[serde(serialize_with = "none_as_empty_string")] pub ssl_cert_path: Option, #[serde(serialize_with = "none_as_empty_string")] @@ -35,8 +34,11 @@ pub struct HttpTrackerConfig { } impl HttpTrackerConfig { - pub fn is_ssl_enabled(&self) -> bool { - self.ssl_enabled && self.ssl_cert_path.is_some() && self.ssl_key_path.is_some() + pub fn verify_ssl_cert_and_key_set(&self) -> bool { + self.ssl_cert_path.is_some() + && self.ssl_key_path.is_some() + && !self.ssl_cert_path.as_ref().unwrap().is_empty() + && !self.ssl_key_path.as_ref().unwrap().is_empty() } } @@ -163,7 +165,6 @@ impl Configuration { enabled: false, bind_address: String::from("0.0.0.0:6969"), ssl_enabled: false, - ssl_bind_address: String::from("0.0.0.0:6868"), ssl_cert_path: None, ssl_key_path: None } @@ -171,15 +172,6 @@ impl Configuration { configuration } - pub fn verify(&self) -> Result<(), ConfigurationError> { - // UDP is not secure for sending private keys - if self.mode == TrackerMode::PrivateMode || self.mode == TrackerMode::PrivateListedMode { - return Err(ConfigurationError::TrackerModeIncompatible) - } - - Ok(()) - } - pub fn load_from_file() -> Result { let mut config = Config::new(); @@ -197,10 +189,7 @@ impl Configuration { let torrust_config: Configuration = config.try_into().map_err(|e| ConfigError::Message(format!("Errors while processing config: {}.", e)))?; - match torrust_config.verify() { - Ok(_) => Ok(torrust_config), - Err(e) => Err(ConfigError::Message(format!("Errors while processing config: {}.", e))) - } + Ok(torrust_config) } pub fn save_to_file(&self) -> Result<(), ()>{ diff --git a/src/main.rs b/src/main.rs index 08610d24a..721385760 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use std::net::SocketAddr; use std::sync::Arc; -use log::info; +use log::{info}; use tokio::task::JoinHandle; use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; use torrust_tracker::torrust_http_tracker::server::HttpServer; @@ -39,24 +39,33 @@ async fn main() { let _api_server = start_api_server(&config.http_api, tracker.clone()); } + // used to send graceful shutdown signal to udp listeners let (tx, rx) = tokio::sync::watch::channel(false); let mut udp_server_handles = Vec::new(); // start the udp blocks for udp_tracker in &config.udp_trackers { - // used to send kill signal to thread + if !udp_tracker.enabled { continue; } - if udp_tracker.enabled { - udp_server_handles.push( - start_udp_tracker_server(&udp_tracker, tracker.clone(), rx.clone()).await - ) + if tracker.is_private() { + panic!("Could not start UDP tracker on: {} while in {:?}. UDP is not safe for private trackers!", udp_tracker.bind_address, config.mode); } + + udp_server_handles.push( + start_udp_tracker_server(&udp_tracker, tracker.clone(), rx.clone()).await + ) } // start the http blocks for http_tracker in &config.http_trackers { - let _ = start_http_tracker_server(&http_tracker, tracker.clone(), true); - let _ = start_http_tracker_server(&http_tracker, tracker.clone(), false); + if !http_tracker.enabled { continue; } + + // SSL requires a cert and a key + if http_tracker.ssl_enabled && !http_tracker.verify_ssl_cert_and_key_set() { + panic!("Could not start HTTP tracker on: {}, missing SSL Cert or Key!", http_tracker.bind_address); + } + + let _ = start_http_tracker_server(&http_tracker, tracker.clone()); } // handle the signals here @@ -110,22 +119,19 @@ fn start_api_server(config: &HttpApiConfig, tracker: Arc) -> Joi }) } -fn start_http_tracker_server(config: &HttpTrackerConfig, tracker: Arc, ssl: bool) -> JoinHandle<()> { +fn start_http_tracker_server(config: &HttpTrackerConfig, tracker: Arc) -> JoinHandle<()> { let http_tracker = HttpServer::new(tracker); - let enabled = config.enabled; let bind_addr = config.bind_address.parse::().unwrap(); let ssl_enabled = config.ssl_enabled; - let ssl_bind_addr = config.ssl_bind_address.parse::().unwrap(); let ssl_cert_path = config.ssl_cert_path.clone(); let ssl_key_path = config.ssl_key_path.clone(); tokio::spawn(async move { // run with tls if ssl_enabled and cert and key path are set - if ssl && ssl_enabled && ssl_cert_path.is_some() && ssl_key_path.is_some() { - info!("Starting HTTPS server on: {} (TLS)", ssl_bind_addr); - http_tracker.start_tls(ssl_bind_addr, ssl_cert_path.as_ref().unwrap(), ssl_key_path.as_ref().unwrap()).await; - } - if !ssl && enabled { + if ssl_enabled { + info!("Starting HTTPS server on: {} (TLS)", bind_addr); + http_tracker.start_tls(bind_addr, ssl_cert_path.as_ref().unwrap(), ssl_key_path.as_ref().unwrap()).await; + } else { info!("Starting HTTP server on: {}", bind_addr); http_tracker.start(bind_addr).await; } diff --git a/src/tracker.rs b/src/tracker.rs index 1caccff23..a1e172690 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -15,7 +15,7 @@ use crate::database::{Database}; use crate::key_manager::Error::KeyInvalid; use crate::torrust_http_tracker::AnnounceRequest; -#[derive(Serialize, Deserialize, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] pub enum TrackerMode { // Will track every new info hash and serve every peer. #[serde(rename = "public")] @@ -294,15 +294,15 @@ impl TorrentTracker { }) } - fn is_public(&self) -> bool { + pub fn is_public(&self) -> bool { self.config.mode == TrackerMode::PublicMode } - fn is_private(&self) -> bool { + pub fn is_private(&self) -> bool { self.config.mode == TrackerMode::PrivateMode || self.config.mode == TrackerMode::PrivateListedMode } - fn is_whitelisted(&self) -> bool { + pub fn is_whitelisted(&self) -> bool { self.config.mode == TrackerMode::ListedMode || self.config.mode == TrackerMode::PrivateListedMode } From 12523c4c4ad05a845cfc46b4f0636a50e06a56bc Mon Sep 17 00:00:00 2001 From: Power2All Date: Tue, 12 Apr 2022 13:06:33 +0200 Subject: [PATCH 05/53] Bypassing a error --- src/tracker.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tracker.rs b/src/tracker.rs index a1e172690..43dd3b00f 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -202,7 +202,9 @@ impl TorrentEntry { } AnnounceEvent::Stopped => { if peer_old.is_seeder() { - self.seeders -= 1; + if self.seeders != 0 { + self.seeders -= 1; + } } } // impossible, started should be the first time a peer announces itself From 64fd27183d3faf20eab36c7dee7c439876c5cbe8 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 13 Apr 2022 01:46:16 +0200 Subject: [PATCH 06/53] fix: fixed PeerKeyInvalid error when using MySQL --- src/mysql_database.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index f6907f309..0597d46aa 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -1,5 +1,5 @@ use std::collections::BTreeMap; -use crate::{InfoHash, AUTH_KEY_LENGTH, TorrentEntry, database}; +use crate::{InfoHash, AUTH_KEY_LENGTH, database}; use log::debug; use r2d2::{Pool}; use crate::key_manager::AuthKey; @@ -9,6 +9,7 @@ use async_trait::async_trait; use r2d2_mysql::mysql::{Opts, OptsBuilder, params, TxOpts}; use r2d2_mysql::mysql::prelude::Queryable; use r2d2_mysql::MysqlConnectionManager; +use crate::torrent::TorrentEntry; pub struct MysqlDatabase { pool: Pool @@ -136,7 +137,7 @@ impl Database for MysqlDatabase { async fn get_key_from_keys(&self, key: &str) -> Result { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - match conn.exec_first::<(String, i64), _, _>("SELECT `key`, valid_until FROM `keys` WHERE `key` = :key", params! { key => key }) + match conn.exec_first::<(String, i64), _, _>("SELECT `key`, valid_until FROM `keys` WHERE `key` = :key", params! { key }) .map_err(|_| database::Error::QueryReturnedNoRows)? { Some((key, valid_until)) => { Ok(AuthKey { From fa330c7d94527613cf36cd47eae0dc0d98f0d481 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 13 Apr 2022 02:14:56 +0200 Subject: [PATCH 07/53] fix: stats tracker now runs on a dedicated thread instead of spawning every request --- Cargo.lock | 1 + Cargo.toml | 2 +- src/database.rs | 3 +- src/http_api_server.rs | 2 +- src/lib.rs | 2 + src/sqlite_database.rs | 3 +- src/torrent.rs | 221 ++++++++++++++++++++++ src/torrust_http_tracker/handlers.rs | 43 ++--- src/torrust_udp_tracker/handlers.rs | 48 ++--- src/tracker.rs | 271 ++------------------------- src/tracker_stats.rs | 123 ++++++++++++ 11 files changed, 400 insertions(+), 319 deletions(-) create mode 100644 src/torrent.rs create mode 100644 src/tracker_stats.rs diff --git a/Cargo.lock b/Cargo.lock index 68f52fa18..2cb824d71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2224,6 +2224,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", + "parking_lot", "pin-project-lite", "signal-hook-registry", "tokio-macros", diff --git a/Cargo.toml b/Cargo.toml index e3e57c378..81f76abe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ serde_json = "1.0.72" hex = "0.4.3" percent-encoding = "2.1.0" warp = {version = "0.3", features = ["tls"]} -tokio = {version = "1.7", features = ["macros", "io-util", "net", "time", "rt-multi-thread", "fs", "sync", "signal"]} +tokio = {version = "1.7", features = ["full"]} binascii = "0.1" toml = "0.5" log = {version = "0.4", features = ["release_max_level_info"]} diff --git a/src/database.rs b/src/database.rs index 0c0e4a303..18bf41994 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,5 +1,5 @@ use std::collections::BTreeMap; -use crate::{InfoHash, TorrentEntry}; +use crate::{InfoHash}; use crate::key_manager::AuthKey; use crate::sqlite_database::SqliteDatabase; use async_trait::async_trait; @@ -7,6 +7,7 @@ use derive_more::{Display, Error}; use log::debug; use crate::mysql_database::MysqlDatabase; use serde::{Serialize, Deserialize}; +use crate::torrent::TorrentEntry; #[derive(Serialize, Deserialize, Debug)] pub enum DatabaseDrivers { diff --git a/src/http_api_server.rs b/src/http_api_server.rs index a6bee4a14..eff45fc33 100644 --- a/src/http_api_server.rs +++ b/src/http_api_server.rs @@ -4,7 +4,7 @@ use std::cmp::min; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use warp::{filters, reply, reply::Reply, serve, Filter, Server}; -use crate::TorrentPeer; +use crate::torrent::TorrentPeer; use super::common::*; #[derive(Deserialize, Debug)] diff --git a/src/lib.rs b/src/lib.rs index addc7e1a1..3d928aff4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,8 @@ pub mod torrust_udp_tracker; pub mod torrust_http_tracker; pub mod database; pub mod mysql_database; +pub mod torrent; +pub mod tracker_stats; pub use self::config::*; pub use torrust_udp_tracker::server::*; diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index c54a3c79c..5facd99d8 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -1,5 +1,5 @@ use std::collections::BTreeMap; -use crate::{InfoHash, AUTH_KEY_LENGTH, TorrentEntry, database}; +use crate::{InfoHash, AUTH_KEY_LENGTH, database}; use log::debug; use r2d2_sqlite::{SqliteConnectionManager}; use r2d2::{Pool}; @@ -8,6 +8,7 @@ use crate::key_manager::AuthKey; use std::str::FromStr; use crate::database::Database; use async_trait::async_trait; +use crate::torrent::TorrentEntry; pub struct SqliteDatabase { pool: Pool diff --git a/src/torrent.rs b/src/torrent.rs new file mode 100644 index 000000000..ef933d224 --- /dev/null +++ b/src/torrent.rs @@ -0,0 +1,221 @@ +use std::borrow::Cow; +use std::net::{IpAddr, SocketAddr}; +use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; +use serde::{Serialize, Deserialize}; +use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; +use crate::torrust_http_tracker::AnnounceRequest; +use crate::common::{AnnounceEventDef, NumberOfBytesDef}; + +#[derive(PartialEq, Eq, Debug, Clone, Serialize)] +pub struct TorrentPeer { + pub peer_id: PeerId, + pub peer_addr: SocketAddr, + #[serde(serialize_with = "ser_instant")] + pub updated: std::time::Instant, + #[serde(with = "NumberOfBytesDef")] + pub uploaded: NumberOfBytes, + #[serde(with = "NumberOfBytesDef")] + pub downloaded: NumberOfBytes, + #[serde(with = "NumberOfBytesDef")] + pub left: NumberOfBytes, + #[serde(with = "AnnounceEventDef")] + pub event: AnnounceEvent, +} + +impl TorrentPeer { + pub fn from_udp_announce_request(announce_request: &aquatic_udp_protocol::AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { + let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port.0); + + TorrentPeer { + peer_id: PeerId(announce_request.peer_id.0), + peer_addr, + updated: std::time::Instant::now(), + uploaded: announce_request.bytes_uploaded, + downloaded: announce_request.bytes_downloaded, + left: announce_request.bytes_left, + event: announce_request.event + } + } + + pub fn from_http_announce_request(announce_request: &AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { + let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port); + + let event: AnnounceEvent = if let Some(event) = &announce_request.event { + match event.as_ref() { + "started" => AnnounceEvent::Started, + "stopped" => AnnounceEvent::Stopped, + "completed" => AnnounceEvent::Completed, + _ => AnnounceEvent::None + } + } else { + AnnounceEvent::None + }; + + TorrentPeer { + peer_id: announce_request.peer_id.clone(), + peer_addr, + updated: std::time::Instant::now(), + uploaded: NumberOfBytes(announce_request.uploaded as i64), + downloaded: NumberOfBytes(announce_request.downloaded as i64), + left: NumberOfBytes(announce_request.left as i64), + event + } + } + + // potentially substitute localhost ip with external ip + pub fn peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip: IpAddr, host_opt_ip: Option, port: u16) -> SocketAddr { + if remote_ip.is_loopback() && host_opt_ip.is_some() { + SocketAddr::new(host_opt_ip.unwrap(), port) + } else { + SocketAddr::new(remote_ip, port) + } + } + + pub(crate) fn is_seeder(&self) -> bool { self.left.0 <= 0 && self.event != AnnounceEvent::Stopped } + + fn is_completed(&self) -> bool { + self.event == AnnounceEvent::Completed + } +} + +fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { + ser.serialize_u64(inst.elapsed().as_millis() as u64) +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct TorrentEntry { + #[serde(skip)] + pub(crate) peers: std::collections::BTreeMap, + pub(crate) completed: u32, + #[serde(skip)] + pub(crate) seeders: u32, +} + +impl TorrentEntry { + pub fn new() -> TorrentEntry { + TorrentEntry { + peers: std::collections::BTreeMap::new(), + completed: 0, + seeders: 0, + } + } + + pub fn update_peer(&mut self, peer: &TorrentPeer) { + match peer.event { + AnnounceEvent::Stopped => { + let peer_old = self.peers.remove(&peer.peer_id); + self.update_torrent_stats_with_peer(peer, peer_old); + } + _ => { + let peer_old = self.peers.insert(peer.peer_id.clone(), peer.clone()); + self.update_torrent_stats_with_peer(peer, peer_old); + } + } + } + + pub fn get_peers(&self, remote_addr: Option<&std::net::SocketAddr>) -> Vec { + let mut list = Vec::new(); + for (_, peer) in self + .peers + .iter() + .filter(|e| match remote_addr { + // don't filter on ip_version + None => true, + // filter out different ip_version from remote_addr + Some(remote_address) => { + match e.1.peer_addr.ip() { + IpAddr::V4(_) => { remote_address.is_ipv4() } + IpAddr::V6(_) => { remote_address.is_ipv6() } + } + } + }) + .take(MAX_SCRAPE_TORRENTS as usize) + { + + // skip ip address of client + if let Some(remote_addr) = remote_addr { + if peer.peer_addr == *remote_addr { + continue; + } + } + + list.push(peer.clone()); + } + list + } + + pub fn update_torrent_stats_with_peer(&mut self, peer: &TorrentPeer, peer_old: Option) { + match peer_old { + None => { + if peer.is_seeder() { + self.seeders += 1; + } + + if peer.is_completed() { + self.completed += 1; + } + } + Some(peer_old) => { + match peer.event { + AnnounceEvent::None => { + if peer.is_seeder() && !peer_old.is_seeder() { + self.seeders += 1; + } + } + AnnounceEvent::Completed => { + if peer.is_seeder() && !peer_old.is_seeder() { + self.seeders += 1; + } + + // don't double count completed + if !peer_old.is_completed() { + self.completed += 1; + } + } + AnnounceEvent::Stopped => { + if peer_old.is_seeder() { + if self.seeders != 0 { + self.seeders -= 1; + } + } + } + // impossible, started should be the first time a peer announces itself + AnnounceEvent::Started => {} + } + } + } + } + + pub fn get_stats(&self) -> (u32, u32, u32) { + let leechers: u32 = if self.seeders < (self.peers.len() as u32) { + (self.peers.len() as u32) - self.seeders + } else { + 0 + }; + + (self.seeders, self.completed, leechers) + } +} + +#[derive(Serialize, Deserialize)] +struct DatabaseRow<'a> { + info_hash: InfoHash, + entry: Cow<'a, TorrentEntry>, +} + +#[derive(Debug)] +pub struct TorrentStats { + pub completed: u32, + pub seeders: u32, + pub leechers: u32, +} + +#[derive(Debug)] +pub enum TorrentError { + TorrentNotWhitelisted, + PeerNotAuthenticated, + PeerKeyNotValid, + NoPeersFound, + CouldNotSendResponse, + InvalidInfoHash, +} diff --git a/src/torrust_http_tracker/handlers.rs b/src/torrust_http_tracker/handlers.rs index f15b7143a..8762faeaf 100644 --- a/src/torrust_http_tracker/handlers.rs +++ b/src/torrust_http_tracker/handlers.rs @@ -1,12 +1,15 @@ use std::collections::HashMap; use std::convert::Infallible; +use std::net::IpAddr; use std::sync::Arc; use log::debug; use warp::{reject, Rejection, Reply}; use warp::http::{Response}; -use crate::{InfoHash, TorrentError, TorrentPeer, TorrentStats, TorrentTracker}; +use crate::{InfoHash, TorrentTracker}; use crate::key_manager::AuthKey; +use crate::torrent::{TorrentError, TorrentPeer, TorrentStats}; use crate::torrust_http_tracker::{AnnounceRequest, AnnounceResponse, ErrorResponse, Peer, ScrapeRequest, ScrapeResponse, ScrapeResponseEntry, ServerError, WebResult}; +use crate::tracker_stats::TrackerStatsEvent; use crate::utils::url_encode_bytes; /// Authenticate InfoHash using optional AuthKey @@ -42,23 +45,14 @@ pub async fn handle_announce(announce_request: AnnounceRequest, auth_key: Option // get all torrent peers excluding the peer_addr let peers = tracker.get_torrent_peers(&announce_request.info_hash, &peer.peer_addr).await; - // success response - let tracker_copy = tracker.clone(); - let is_ipv4 = announce_request.peer_addr.is_ipv4(); - - tokio::spawn(async move { - let mut status_writer = tracker_copy.set_stats().await; - if is_ipv4 { - status_writer.tcp4_connections_handled += 1; - status_writer.tcp4_announces_handled += 1; - } else { - status_writer.tcp6_connections_handled += 1; - status_writer.tcp6_announces_handled += 1; - } - }); - let announce_interval = tracker.config.announce_interval; + // send stats event + match announce_request.peer_addr { + IpAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp4Announce).await; } + IpAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp6Announce).await; } + } + send_announce_response(&announce_request, torrent_stats, peers, announce_interval, tracker.config.announce_interval_min) } @@ -86,18 +80,11 @@ pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp4Scrape).await; } + IpAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp6Scrape).await; } + } send_scrape_response(files) } diff --git a/src/torrust_udp_tracker/handlers.rs b/src/torrust_udp_tracker/handlers.rs index bf25a8861..df1a15451 100644 --- a/src/torrust_udp_tracker/handlers.rs +++ b/src/torrust_udp_tracker/handlers.rs @@ -1,9 +1,11 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use aquatic_udp_protocol::{AnnounceInterval, AnnounceRequest, AnnounceResponse, ConnectRequest, ConnectResponse, ErrorResponse, NumberOfDownloads, NumberOfPeers, Port, Request, Response, ResponsePeer, ScrapeRequest, ScrapeResponse, TorrentScrapeStatistics, TransactionId}; -use crate::{InfoHash, MAX_SCRAPE_TORRENTS, TorrentError, TorrentPeer, TorrentTracker}; +use crate::{InfoHash, MAX_SCRAPE_TORRENTS, TorrentTracker}; +use crate::torrent::{TorrentError, TorrentPeer}; use crate::torrust_udp_tracker::errors::ServerError; use crate::torrust_udp_tracker::request::AnnounceRequestWrapper; +use crate::tracker_stats::TrackerStatsEvent; use crate::utils::get_connection_id; pub async fn authenticate(info_hash: &InfoHash, tracker: Arc) -> Result<(), ServerError> { @@ -71,15 +73,11 @@ pub async fn handle_connect(remote_addr: SocketAddr, request: &ConnectRequest, t connection_id, }); - let tracker_copy = tracker.clone(); - tokio::spawn(async move { - let mut status_writer = tracker_copy.set_stats().await; - if remote_addr.is_ipv4() { - status_writer.udp4_connections_handled += 1; - } else { - status_writer.udp6_connections_handled += 1; - } - }); + // send stats event + match remote_addr { + SocketAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp4Connect).await; } + SocketAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp6Connect).await; } + } Ok(response) } @@ -98,16 +96,6 @@ pub async fn handle_announce(remote_addr: SocketAddr, announce_request: &Announc // get all peers excluding the client_addr let peers = tracker.get_torrent_peers(&wrapped_announce_request.info_hash, &peer.peer_addr).await; - let tracker_copy = tracker.clone(); - tokio::spawn(async move { - let mut status_writer = tracker_copy.set_stats().await; - if remote_addr.is_ipv4() { - status_writer.udp4_announces_handled += 1; - } else { - status_writer.udp6_announces_handled += 1; - } - }); - let announce_response = if remote_addr.is_ipv4() { Response::from(AnnounceResponse { transaction_id: wrapped_announce_request.announce_request.transaction_id, @@ -144,6 +132,12 @@ pub async fn handle_announce(remote_addr: SocketAddr, announce_request: &Announc }) }; + // send stats event + match remote_addr { + SocketAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp4Announce).await; } + SocketAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp6Announce).await; } + } + Ok(announce_response) } @@ -180,15 +174,11 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra torrent_stats.push(scrape_entry); } - let tracker_copy = tracker.clone(); - tokio::spawn(async move { - let mut status_writer = tracker_copy.set_stats().await; - if remote_addr.is_ipv4() { - status_writer.udp4_scrapes_handled += 1; - } else { - status_writer.udp6_scrapes_handled += 1; - } - }); + // send stats event + match remote_addr { + SocketAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp4Scrape).await; } + SocketAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp6Scrape).await; } + } Ok(Response::from(ScrapeResponse { transaction_id: request.transaction_id, diff --git a/src/tracker.rs b/src/tracker.rs index 43dd3b00f..eb6f006bf 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,19 +1,18 @@ use serde::{Deserialize, Serialize}; use serde; -use std::borrow::Cow; use std::collections::BTreeMap; -use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use crate::common::{AnnounceEventDef, InfoHash, NumberOfBytesDef, PeerId}; -use std::net::{IpAddr, SocketAddr}; -use crate::{Configuration, database, key_manager, MAX_SCRAPE_TORRENTS}; +use tokio::sync::{RwLock, RwLockReadGuard}; +use crate::common::{InfoHash}; +use std::net::{SocketAddr}; +use crate::{Configuration, database, key_manager}; use std::collections::btree_map::Entry; use std::sync::Arc; -use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; use log::info; use crate::key_manager::AuthKey; use crate::database::{Database}; use crate::key_manager::Error::KeyInvalid; -use crate::torrust_http_tracker::AnnounceRequest; +use crate::torrent::{TorrentEntry, TorrentError, TorrentPeer, TorrentStats}; +use crate::tracker_stats::{StatsTracker, TrackerStats}; #[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] pub enum TrackerMode { @@ -34,265 +33,25 @@ pub enum TrackerMode { PrivateListedMode, } -#[derive(PartialEq, Eq, Debug, Clone, Serialize)] -pub struct TorrentPeer { - pub peer_id: PeerId, - pub peer_addr: SocketAddr, - #[serde(serialize_with = "ser_instant")] - pub updated: std::time::Instant, - #[serde(with = "NumberOfBytesDef")] - pub uploaded: NumberOfBytes, - #[serde(with = "NumberOfBytesDef")] - pub downloaded: NumberOfBytes, - #[serde(with = "NumberOfBytesDef")] - pub left: NumberOfBytes, - #[serde(with = "AnnounceEventDef")] - pub event: AnnounceEvent, -} - -impl TorrentPeer { - pub fn from_udp_announce_request(announce_request: &aquatic_udp_protocol::AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { - let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port.0); - - TorrentPeer { - peer_id: PeerId(announce_request.peer_id.0), - peer_addr, - updated: std::time::Instant::now(), - uploaded: announce_request.bytes_uploaded, - downloaded: announce_request.bytes_downloaded, - left: announce_request.bytes_left, - event: announce_request.event - } - } - - pub fn from_http_announce_request(announce_request: &AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { - let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port); - - let event: AnnounceEvent = if let Some(event) = &announce_request.event { - match event.as_ref() { - "started" => AnnounceEvent::Started, - "stopped" => AnnounceEvent::Stopped, - "completed" => AnnounceEvent::Completed, - _ => AnnounceEvent::None - } - } else { - AnnounceEvent::None - }; - - TorrentPeer { - peer_id: announce_request.peer_id.clone(), - peer_addr, - updated: std::time::Instant::now(), - uploaded: NumberOfBytes(announce_request.uploaded as i64), - downloaded: NumberOfBytes(announce_request.downloaded as i64), - left: NumberOfBytes(announce_request.left as i64), - event - } - } - - // potentially substitute localhost ip with external ip - pub fn peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip: IpAddr, host_opt_ip: Option, port: u16) -> SocketAddr { - if remote_ip.is_loopback() && host_opt_ip.is_some() { - SocketAddr::new(host_opt_ip.unwrap(), port) - } else { - SocketAddr::new(remote_ip, port) - } - } - - fn is_seeder(&self) -> bool { self.left.0 <= 0 && self.event != AnnounceEvent::Stopped } - - fn is_completed(&self) -> bool { - self.event == AnnounceEvent::Completed - } -} - -fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { - ser.serialize_u64(inst.elapsed().as_millis() as u64) -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct TorrentEntry { - #[serde(skip)] - peers: std::collections::BTreeMap, - completed: u32, - #[serde(skip)] - seeders: u32, -} - -impl TorrentEntry { - pub fn new() -> TorrentEntry { - TorrentEntry { - peers: std::collections::BTreeMap::new(), - completed: 0, - seeders: 0, - } - } - - pub fn update_peer(&mut self, peer: &TorrentPeer) { - match peer.event { - AnnounceEvent::Stopped => { - let peer_old = self.peers.remove(&peer.peer_id); - self.update_torrent_stats_with_peer(peer, peer_old); - } - _ => { - let peer_old = self.peers.insert(peer.peer_id.clone(), peer.clone()); - self.update_torrent_stats_with_peer(peer, peer_old); - } - } - } - - pub fn get_peers(&self, remote_addr: Option<&std::net::SocketAddr>) -> Vec { - let mut list = Vec::new(); - for (_, peer) in self - .peers - .iter() - .filter(|e| match remote_addr { - // don't filter on ip_version - None => true, - // filter out different ip_version from remote_addr - Some(remote_address) => { - match e.1.peer_addr.ip() { - IpAddr::V4(_) => { remote_address.is_ipv4() } - IpAddr::V6(_) => { remote_address.is_ipv6() } - } - } - }) - .take(MAX_SCRAPE_TORRENTS as usize) - { - - // skip ip address of client - if let Some(remote_addr) = remote_addr { - if peer.peer_addr == *remote_addr { - continue; - } - } - - list.push(peer.clone()); - } - list - } - - pub fn update_torrent_stats_with_peer(&mut self, peer: &TorrentPeer, peer_old: Option) { - match peer_old { - None => { - if peer.is_seeder() { - self.seeders += 1; - } - - if peer.is_completed() { - self.completed += 1; - } - } - Some(peer_old) => { - match peer.event { - AnnounceEvent::None => { - if peer.is_seeder() && !peer_old.is_seeder() { - self.seeders += 1; - } - } - AnnounceEvent::Completed => { - if peer.is_seeder() && !peer_old.is_seeder() { - self.seeders += 1; - } - - // don't double count completed - if !peer_old.is_completed() { - self.completed += 1; - } - } - AnnounceEvent::Stopped => { - if peer_old.is_seeder() { - if self.seeders != 0 { - self.seeders -= 1; - } - } - } - // impossible, started should be the first time a peer announces itself - AnnounceEvent::Started => {} - } - } - } - } - - pub fn get_stats(&self) -> (u32, u32, u32) { - let leechers: u32 = if self.seeders < (self.peers.len() as u32) { - (self.peers.len() as u32) - self.seeders - } else { - 0 - }; - - (self.seeders, self.completed, leechers) - } -} - -#[derive(Serialize, Deserialize)] -struct DatabaseRow<'a> { - info_hash: InfoHash, - entry: Cow<'a, TorrentEntry>, -} - -#[derive(Debug)] -pub struct TorrentStats { - pub completed: u32, - pub seeders: u32, - pub leechers: u32, -} - -#[derive(Debug)] -pub enum TorrentError { - TorrentNotWhitelisted, - PeerNotAuthenticated, - PeerKeyNotValid, - NoPeersFound, - CouldNotSendResponse, - InvalidInfoHash, -} - -#[derive(Debug)] -pub struct TrackerStats { - pub tcp4_connections_handled: u64, - pub tcp4_announces_handled: u64, - pub tcp4_scrapes_handled: u64, - pub tcp6_connections_handled: u64, - pub tcp6_announces_handled: u64, - pub tcp6_scrapes_handled: u64, - pub udp4_connections_handled: u64, - pub udp4_announces_handled: u64, - pub udp4_scrapes_handled: u64, - pub udp6_connections_handled: u64, - pub udp6_announces_handled: u64, - pub udp6_scrapes_handled: u64, -} - pub struct TorrentTracker { pub config: Arc, torrents: tokio::sync::RwLock>, database: Box, - stats: tokio::sync::RwLock, + pub stats_tracker: StatsTracker } impl TorrentTracker { pub fn new(config: Arc) -> Result { let database = database::connect_database(&config.db_driver, &config.db_path)?; + let mut stats_tracker = StatsTracker::new(); + + stats_tracker.run_worker(); Ok(TorrentTracker { config, torrents: RwLock::new(std::collections::BTreeMap::new()), database, - stats: RwLock::new(TrackerStats { - tcp4_connections_handled: 0, - tcp4_announces_handled: 0, - tcp4_scrapes_handled: 0, - tcp6_connections_handled: 0, - tcp6_announces_handled: 0, - tcp6_scrapes_handled: 0, - udp4_connections_handled: 0, - udp4_announces_handled: 0, - udp4_scrapes_handled: 0, - udp6_connections_handled: 0, - udp6_announces_handled: 0, - udp6_scrapes_handled: 0, - }), + stats_tracker }) } @@ -359,7 +118,7 @@ impl TorrentTracker { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { - self.add_torrent(torrent.0, 0, torrent.1, 0).await; + let _ = self.add_torrent(torrent.0, 0, torrent.1, 0).await; } Ok(()) @@ -449,12 +208,8 @@ impl TorrentTracker { self.torrents.read().await } - pub async fn set_stats(&self) -> RwLockWriteGuard<'_, TrackerStats> { - self.stats.write().await - } - pub async fn get_stats(&self) -> RwLockReadGuard<'_, TrackerStats> { - self.stats.read().await + self.stats_tracker.get_stats().await } // remove torrents without peers if enabled, and defragment memory diff --git a/src/tracker_stats.rs b/src/tracker_stats.rs new file mode 100644 index 000000000..1a6a71c2b --- /dev/null +++ b/src/tracker_stats.rs @@ -0,0 +1,123 @@ +use std::sync::Arc; +use tokio::sync::{mpsc, RwLock, RwLockReadGuard}; +use tokio::sync::mpsc::{Sender}; +use tokio::sync::mpsc::error::SendError; + +const CHANNEL_BUFFER_SIZE: usize = 65_535; + +#[derive(Debug)] +pub enum TrackerStatsEvent { + Tcp4Announce, + Tcp4Scrape, + Tcp6Announce, + Tcp6Scrape, + Udp4Connect, + Udp4Announce, + Udp4Scrape, + Udp6Connect, + Udp6Announce, + Udp6Scrape +} + +#[derive(Debug)] +pub struct TrackerStats { + pub tcp4_connections_handled: u64, + pub tcp4_announces_handled: u64, + pub tcp4_scrapes_handled: u64, + pub tcp6_connections_handled: u64, + pub tcp6_announces_handled: u64, + pub tcp6_scrapes_handled: u64, + pub udp4_connections_handled: u64, + pub udp4_announces_handled: u64, + pub udp4_scrapes_handled: u64, + pub udp6_connections_handled: u64, + pub udp6_announces_handled: u64, + pub udp6_scrapes_handled: u64, +} + +impl TrackerStats { + pub fn new() -> Self { + Self { + tcp4_connections_handled: 0, + tcp4_announces_handled: 0, + tcp4_scrapes_handled: 0, + tcp6_connections_handled: 0, + tcp6_announces_handled: 0, + tcp6_scrapes_handled: 0, + udp4_connections_handled: 0, + udp4_announces_handled: 0, + udp4_scrapes_handled: 0, + udp6_connections_handled: 0, + udp6_announces_handled: 0, + udp6_scrapes_handled: 0, + } + } +} + +pub struct StatsTracker { + channel_sender: Option>, + pub stats: Arc> +} + +impl StatsTracker { + pub fn new() -> Self { + Self { + channel_sender: None, + stats: Arc::new(RwLock::new(TrackerStats::new())) + } + } + + pub async fn get_stats(&self) -> RwLockReadGuard<'_, TrackerStats> { + self.stats.read().await + } + + pub async fn send_event(&self, event: TrackerStatsEvent) -> Option>> { + if let Some(tx) = &self.channel_sender { + Some(tx.send(event).await) + } else { + None + } + } + + pub fn run_worker(&mut self) { + let (tx, mut rx) = mpsc::channel::(CHANNEL_BUFFER_SIZE); + + // set send channel on stats_tracker + self.channel_sender = Some(tx); + + let stats = self.stats.clone(); + + tokio::spawn(async move { + while let Some(event) = rx.recv().await { + let mut stats_lock = stats.write().await; + + match event { + TrackerStatsEvent::Tcp4Announce => { + stats_lock.tcp4_announces_handled += 1; + stats_lock.tcp4_connections_handled += 1; + } + TrackerStatsEvent::Tcp4Scrape => { + stats_lock.tcp4_scrapes_handled += 1; + stats_lock.tcp4_connections_handled += 1; + } + TrackerStatsEvent::Tcp6Announce => { + stats_lock.tcp6_announces_handled += 1; + stats_lock.tcp6_connections_handled += 1; + } + TrackerStatsEvent::Tcp6Scrape => { + stats_lock.tcp6_scrapes_handled += 1; + stats_lock.tcp6_connections_handled += 1; + } + TrackerStatsEvent::Udp4Connect => { stats_lock.udp4_connections_handled += 1; } + TrackerStatsEvent::Udp4Announce => { stats_lock.udp4_announces_handled += 1; } + TrackerStatsEvent::Udp4Scrape => { stats_lock.udp4_scrapes_handled += 1; } + TrackerStatsEvent::Udp6Connect => { stats_lock.udp6_connections_handled += 1; } + TrackerStatsEvent::Udp6Announce => { stats_lock.udp6_announces_handled += 1; } + TrackerStatsEvent::Udp6Scrape => { stats_lock.udp6_scrapes_handled += 1; } + } + + drop(stats_lock); + } + }); + } +} From 6cec6f2a32ae05104183e5c0e9d098799b6345a8 Mon Sep 17 00:00:00 2001 From: Power2All Date: Wed, 13 Apr 2022 14:53:23 +0200 Subject: [PATCH 08/53] Adding periodic --- src/config.rs | 2 ++ src/main.rs | 22 ++++++++++++++++++++ src/tracker.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index f9166e577..3045c1ca9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,6 +56,7 @@ pub struct Configuration { pub db_driver: DatabaseDrivers, pub db_path: String, pub persistence: bool, + pub persistence_interval: Option, pub cleanup_interval: Option, pub cleanup_peerless: bool, pub external_ip: Option, @@ -139,6 +140,7 @@ impl Configuration { db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), persistence: false, + persistence_interval: Some(900), cleanup_interval: Some(600), cleanup_peerless: true, external_ip: Some(String::from("0.0.0.0")), diff --git a/src/main.rs b/src/main.rs index 721385760..c3d9ba23c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,8 @@ async fn main() { panic!("Could not load persistent torrents.") }; info!("Persistent torrents loaded."); + + let _torrent_periodic_job = start_torrent_periodic_job(config.clone(), tracker.clone()).unwrap(); } // start torrent cleanup job (periodically removes old peers) @@ -89,6 +91,26 @@ async fn main() { } } +fn start_torrent_periodic_job(config: Arc, tracker: Arc) -> Option> { + let weak_tracker = std::sync::Arc::downgrade(&tracker); + let interval = config.persistence_interval.unwrap_or(900); + + return Some(tokio::spawn(async move { + let interval = std::time::Duration::from_secs(interval); + let mut interval = tokio::time::interval(interval); + interval.tick().await; // first tick is immediate... + // periodically call tracker.cleanup_torrents() + loop { + interval.tick().await; + if let Some(tracker) = weak_tracker.upgrade() { + tracker.periodic_saving().await; + } else { + break; + } + } + })); +} + fn start_torrent_cleanup_job(config: Arc, tracker: Arc) -> Option> { let weak_tracker = std::sync::Arc::downgrade(&tracker); let interval = config.cleanup_interval.unwrap_or(600); diff --git a/src/tracker.rs b/src/tracker.rs index eb6f006bf..b2a65b4ce 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; use serde; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use tokio::sync::{RwLock, RwLockReadGuard}; use crate::common::{InfoHash}; use std::net::{SocketAddr}; @@ -36,6 +36,8 @@ pub enum TrackerMode { pub struct TorrentTracker { pub config: Arc, torrents: tokio::sync::RwLock>, + updates: tokio::sync::RwLock>, + shadow: tokio::sync::RwLock>, database: Box, pub stats_tracker: StatsTracker } @@ -50,6 +52,8 @@ impl TorrentTracker { Ok(TorrentTracker { config, torrents: RwLock::new(std::collections::BTreeMap::new()), + updates: RwLock::new(std::collections::HashMap::new()), + shadow: RwLock::new(std::collections::HashMap::new()), database, stats_tracker }) @@ -178,6 +182,15 @@ impl TorrentTracker { let (seeders, completed, leechers) = torrent_entry.get_stats(); + if self.config.persistence { + let mut updates = self.updates.write().await; + if updates.contains_key(info_hash) { + updates.remove(info_hash); + } + updates.insert(*info_hash, completed); + drop(updates); + } + TorrentStats { seeders, leechers, @@ -261,4 +274,43 @@ impl TorrentTracker { } info!("Torrents cleaned up."); } + + pub async fn periodic_saving(&self) { + // Get a lock for writing + let mut shadow = self.shadow.write().await; + + // We will get the data and insert it into the shadow, while clearing updates. + let mut updates = self.updates.write().await; + + for (infohash, completed) in updates.iter() { + if shadow.contains_key(infohash) { + shadow.remove(infohash); + } + shadow.insert(*infohash, *completed); + } + updates.clear(); + drop(updates); + + // We get shadow data into local array to be handled. + let mut shadow_copy: BTreeMap = BTreeMap::new(); + for (infohash, completed) in shadow.iter() { + shadow_copy.insert(*infohash, TorrentEntry{ + peers: Default::default(), + completed: *completed, + seeders: 0 + }); + } + + // Drop the lock + drop(shadow); + + // We will now save the data from the shadow into the database. + // This should not put any strain on the server itself, other then the harddisk/ssd. + let result = self.database.save_persistent_torrent_data(&shadow_copy).await; + if result.is_ok() { + let mut shadow = self.shadow.write().await; + shadow.clear(); + drop(shadow); + } + } } From f7b6c4e0f4914a30b69ef193a18a671d5a3509e2 Mon Sep 17 00:00:00 2001 From: Power2All Date: Wed, 13 Apr 2022 15:10:30 +0200 Subject: [PATCH 09/53] Fix the saving... --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c3d9ba23c..ca632177d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -84,7 +84,7 @@ async fn main() { // Save torrents if enabled if config.persistence { info!("Saving torrents into SQL from memory..."); - let _ = tracker.save_torrents().await; + let _ = tracker.periodic_saving().await; info!("Torrents saved"); } } From f581289b48a96773eab9676fd0db82c7a1890b50 Mon Sep 17 00:00:00 2001 From: Power2All Date: Wed, 13 Apr 2022 15:34:56 +0200 Subject: [PATCH 10/53] Showing information that it's saving periodically --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index ca632177d..2c5ed8678 100644 --- a/src/main.rs +++ b/src/main.rs @@ -103,7 +103,9 @@ fn start_torrent_periodic_job(config: Arc, tracker: Arc Date: Wed, 13 Apr 2022 16:47:22 +0200 Subject: [PATCH 11/53] Adding memory logging --- src/config.rs | 2 ++ src/main.rs | 23 +++++++++++++++++++++++ src/tracker.rs | 16 +++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 3045c1ca9..67078d608 100644 --- a/src/config.rs +++ b/src/config.rs @@ -52,6 +52,7 @@ pub struct HttpApiConfig { #[derive(Serialize, Deserialize)] pub struct Configuration { pub log_level: Option, + pub log_interval: Option, pub mode: TrackerMode, pub db_driver: DatabaseDrivers, pub db_path: String, @@ -136,6 +137,7 @@ impl Configuration { pub fn default() -> Configuration { let mut configuration = Configuration { log_level: Option::from(String::from("info")), + log_interval: Some(60), mode: TrackerMode::PublicMode, db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), diff --git a/src/main.rs b/src/main.rs index 2c5ed8678..9ba45427e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,9 @@ async fn main() { let _ = start_http_tracker_server(&http_tracker, tracker.clone()); } + // start a thread to post statistics + let _ = start_statistics_job(config.clone(), tracker.clone()).unwrap(); + // handle the signals here tokio::select! { _ = tokio::signal::ctrl_c() => { @@ -133,6 +136,26 @@ fn start_torrent_cleanup_job(config: Arc, tracker: Arc, tracker: Arc) -> Option> { + let weak_tracker = std::sync::Arc::downgrade(&tracker); + let interval = config.log_interval.unwrap_or(60); + + return Some(tokio::spawn(async move { + let interval = std::time::Duration::from_secs(interval); + let mut interval = tokio::time::interval(interval); + interval.tick().await; // first tick is immediate... + // periodically call tracker.cleanup_torrents() + loop { + interval.tick().await; + if let Some(tracker) = weak_tracker.upgrade() { + tracker.post_log().await; + } else { + break; + } + } + })); +} + fn start_api_server(config: &HttpApiConfig, tracker: Arc) -> JoinHandle<()> { info!("Starting HTTP API server on: {}", config.bind_address); let bind_addr = config.bind_address.parse::().unwrap(); diff --git a/src/tracker.rs b/src/tracker.rs index b2a65b4ce..68243a02e 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,11 +1,12 @@ use serde::{Deserialize, Serialize}; use serde; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use tokio::sync::{RwLock, RwLockReadGuard}; use crate::common::{InfoHash}; use std::net::{SocketAddr}; use crate::{Configuration, database, key_manager}; use std::collections::btree_map::Entry; +use std::mem; use std::sync::Arc; use log::info; use crate::key_manager::AuthKey; @@ -225,6 +226,19 @@ impl TorrentTracker { self.stats_tracker.get_stats().await } + pub async fn post_log(&self) { + let torrents = self.torrents.read().await; + let torrents_size = em::size_of_val(&*torrents); + drop(torrents); + let updates = self.updates.read().await; + let updates_size = em::size_of_val(&*updates); + drop(updates); + let shadow = self.shadow.read().await; + let shadow_size = em::size_of_val(&*shadow); + drop(shadow); + info!("Stats [::] Torrents: {} byte(s) | Updates: {} byte(s) | Shadow: {} byte(s)", torrents_size, updates_size, shadow_size); + } + // remove torrents without peers if enabled, and defragment memory pub async fn cleanup_torrents(&self) { info!("Cleaning torrents..."); From 41785b62ef7a000b0278f033c629e889c2a738f6 Mon Sep 17 00:00:00 2001 From: Power2All Date: Wed, 13 Apr 2022 16:49:21 +0200 Subject: [PATCH 12/53] Typo ... --- src/tracker.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tracker.rs b/src/tracker.rs index 68243a02e..4aaa767be 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -228,13 +228,13 @@ impl TorrentTracker { pub async fn post_log(&self) { let torrents = self.torrents.read().await; - let torrents_size = em::size_of_val(&*torrents); + let torrents_size = mem::size_of_val(&*torrents); drop(torrents); let updates = self.updates.read().await; - let updates_size = em::size_of_val(&*updates); + let updates_size = mem::size_of_val(&*updates); drop(updates); let shadow = self.shadow.read().await; - let shadow_size = em::size_of_val(&*shadow); + let shadow_size = mem::size_of_val(&*shadow); drop(shadow); info!("Stats [::] Torrents: {} byte(s) | Updates: {} byte(s) | Shadow: {} byte(s)", torrents_size, updates_size, shadow_size); } From c6dff90018d6b2a317cd493da9ebadcd40e3815e Mon Sep 17 00:00:00 2001 From: Power2All Date: Wed, 13 Apr 2022 19:41:58 +0200 Subject: [PATCH 13/53] Adding a statistics viewing in console --- Cargo.lock | 764 ++++++++++++++++++++++++------------------------- src/tracker.rs | 15 +- 2 files changed, 381 insertions(+), 398 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cb824d71..7bc7233c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,11 +10,11 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.3", + "getrandom", "once_cell", "version_check", ] @@ -39,8 +39,8 @@ dependencies = [ [[package]] name = "aquatic_udp_protocol" -version = "0.1.0" -source = "git+https://github.com/greatest-ape/aquatic#065e007ede84de20f20983b4b504471bbda2fdf2" +version = "0.2.0" +source = "git+https://github.com/greatest-ape/aquatic#26e2e874377a2682f52568f8e5e8c080c3366326" dependencies = [ "byteorder", "either", @@ -60,9 +60,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ "proc-macro2", "quote", @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base-x" @@ -107,7 +107,7 @@ dependencies = [ "num-bigint 0.3.3", "num-integer", "num-traits 0.2.14", - "serde 1.0.120", + "serde 1.0.136", ] [[package]] @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitvec" @@ -157,34 +157,22 @@ dependencies = [ "wyz", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array 0.12.3", -] - [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.4", + "generic-array", ] [[package]] -name = "block-padding" -version = "0.1.5" +name = "block-buffer" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ - "byte-tools", + "generic-array", ] [[package]] @@ -205,27 +193,21 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" [[package]] name = "bumpalo" -version = "3.8.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" - -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cc" @@ -242,12 +224,6 @@ dependencies = [ "nom", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -263,7 +239,7 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.14", - "serde 1.0.120", + "serde 1.0.136", "time 0.1.44", "winapi", ] @@ -312,7 +288,7 @@ dependencies = [ "lazy_static", "nom", "rust-ini", - "serde 1.0.120", + "serde 1.0.136", "serde-hjson", "serde_json", "toml", @@ -349,9 +325,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] @@ -362,7 +338,17 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", ] [[package]] @@ -391,20 +377,21 @@ dependencies = [ [[package]] name = "digest" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.12.3", + "generic-array", ] [[package]] name = "digest" -version = "0.9.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "generic-array 0.14.4", + "block-buffer 0.10.2", + "crypto-common", ] [[package]] @@ -432,12 +419,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -450,6 +431,15 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "fern" version = "0.6.0" @@ -461,11 +451,11 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crc32fast", "libc", "libz-sys", @@ -495,9 +485,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", "percent-encoding", @@ -664,18 +654,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.12.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" -dependencies = [ - "typenum", -] - -[[package]] -name = "generic-array" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" dependencies = [ "typenum", "version_check", @@ -683,22 +664,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.10.0+wasi-snapshot-preview1", ] @@ -711,9 +681,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.4" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -724,16 +694,10 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.1", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" - [[package]] name = "hashbrown" version = "0.11.2" @@ -745,18 +709,18 @@ dependencies = [ [[package]] name = "headers" -version = "0.3.3" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62689dc57c7456e69712607ffcbd0aa1dfcccf9af73727e9b25bc1825375cac3" +checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d" dependencies = [ "base64", "bitflags", "bytes", "headers-core", "http", + "httpdate", "mime", - "sha-1 0.8.2", - "time 0.1.44", + "sha-1 0.10.0", ] [[package]] @@ -770,9 +734,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] @@ -785,9 +749,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", @@ -796,25 +760,26 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" dependencies = [ "bytes", "http", + "pin-project-lite", ] [[package]] name = "httparse" -version = "1.5.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" +checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" [[package]] name = "httpdate" -version = "0.3.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "humantime" @@ -824,9 +789,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.2" +version = "0.14.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" dependencies = [ "bytes", "futures-channel", @@ -838,8 +803,8 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project", - "socket2 0.3.19", + "pin-project-lite", + "socket2", "tokio", "tower-service", "tracing", @@ -859,30 +824,21 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", - "hashbrown 0.9.1", -] - -[[package]] -name = "input_buffer" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" -dependencies = [ - "bytes", + "hashbrown", ] [[package]] name = "instant" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -899,15 +855,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.7" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "js-sys" -version = "0.3.55" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] @@ -930,7 +886,7 @@ version = "5.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f404a90a744e32e8be729034fc33b90cf2a56418fbf594d69aa3c0214ad414e5" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "lexical-core", ] @@ -942,16 +898,16 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec 0.5.2", "bitflags", - "cfg-if 1.0.0", + "cfg-if", "ryu", "static_assertions", ] [[package]] name = "libc" -version = "0.2.120" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09" +checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd" [[package]] name = "libloading" @@ -959,7 +915,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "winapi", ] @@ -992,20 +948,21 @@ checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "lock_api" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" +checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ - "cfg-if 0.1.10", + "cfg-if", ] [[package]] @@ -1014,7 +971,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ea2d928b485416e8908cff2d97d621db22b27f7b3b6729e438bcf42c671ba91" dependencies = [ - "hashbrown 0.11.2", + "hashbrown", ] [[package]] @@ -1028,9 +985,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "memchr" @@ -1055,9 +1012,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mime_guess" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" dependencies = [ "mime", "unicase", @@ -1065,42 +1022,41 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" dependencies = [ "adler", - "autocfg", ] [[package]] name = "mio" -version = "0.7.7" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" dependencies = [ "libc", "log", "miow", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi", ] [[package]] name = "miow" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "socket2 0.3.19", "winapi", ] [[package]] name = "multipart" -version = "0.17.1" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050aeedc89243f5347c3e237e3e13dc76fbe4ae3742a57b94dc14f69acf76d4" +checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" dependencies = [ "buf_redux", "httparse", @@ -1108,7 +1064,7 @@ dependencies = [ "mime", "mime_guess", "quick-error", - "rand 0.7.3", + "rand", "safemem", "tempfile", "twoway", @@ -1132,9 +1088,9 @@ dependencies = [ "once_cell", "pem", "percent-encoding", - "serde 1.0.120", + "serde 1.0.136", "serde_json", - "socket2 0.4.4", + "socket2", "twox-hash", "url", ] @@ -1162,11 +1118,11 @@ dependencies = [ "lexical", "num-bigint 0.4.3", "num-traits 0.2.14", - "rand 0.8.4", + "rand", "regex", "rust_decimal", "saturating", - "serde 1.0.120", + "serde 1.0.136", "serde_json", "sha1", "sha2", @@ -1188,9 +1144,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -1212,7 +1168,7 @@ checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" dependencies = [ "bitflags", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "memoffset", ] @@ -1230,9 +1186,9 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" dependencies = [ "winapi", ] @@ -1289,9 +1245,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -1303,12 +1259,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.0" @@ -1322,7 +1272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -1350,22 +1300,32 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.2", ] [[package]] name = "parking_lot_core" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "instant", "libc", "redox_syscall", @@ -1373,6 +1333,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -1398,18 +1371,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.4" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.4" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" dependencies = [ "proc-macro2", "quote", @@ -1418,9 +1391,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.4" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" +checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" [[package]] name = "pin-utils" @@ -1430,15 +1403,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "ppv-lite86" -version = "0.2.10" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro-hack" @@ -1448,9 +1421,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] @@ -1463,9 +1436,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1477,7 +1450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ "log", - "parking_lot", + "parking_lot 0.11.2", "scheduled-thread-pool", ] @@ -1509,37 +1482,13 @@ checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" [[package]] name = "rand" -version = "0.7.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "getrandom 0.1.16", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", -] - -[[package]] -name = "rand" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.3", - "rand_hc 0.3.1", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -1549,16 +1498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -1567,41 +1507,23 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.3", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core 0.6.3", + "getrandom", ] [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ "aho-corasick", "memchr", @@ -1662,13 +1584,13 @@ checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" [[package]] name = "rust_decimal" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37baa70cf8662d2ba1c1868c5983dda16ef32b105cce41fb5c47e72936a90b3" +checksum = "22dc69eadbf0ee2110b8d20418c0c6edbaefec2811c4963dc17b6344e11fe0f8" dependencies = [ "arrayvec 0.7.2", "num-traits 0.2.14", - "serde 1.0.120", + "serde 1.0.136", ] [[package]] @@ -1692,7 +1614,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.4", + "semver 1.0.7", ] [[package]] @@ -1710,9 +1632,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" [[package]] name = "safemem" @@ -1742,7 +1664,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" dependencies = [ - "parking_lot", + "parking_lot 0.11.2", ] [[package]] @@ -1769,9 +1691,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.3.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags", "core-foundation", @@ -1801,9 +1723,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.4" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" [[package]] name = "semver-parser" @@ -1819,9 +1741,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.120" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166b2349061381baf54a58e4b13c89369feb0ef2eaa57198899e2312aac30aab" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" dependencies = [ "serde_derive", ] @@ -1844,7 +1766,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "934d8bdbaa0126dafaea9a8833424a211d9661897717846c6bb782349ca1c30d" dependencies = [ - "serde 1.0.120", + "serde 1.0.136", "serde_bytes", ] @@ -1854,14 +1776,14 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" dependencies = [ - "serde 1.0.120", + "serde 1.0.136", ] [[package]] name = "serde_derive" -version = "1.0.120" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca2a8cb5805ce9e3b95435e3765b7b553cecc762d938d409434338386cb5775" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" dependencies = [ "proc-macro2", "quote", @@ -1870,50 +1792,49 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.72" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ "itoa", "ryu", - "serde 1.0.120", + "serde 1.0.136", ] [[package]] name = "serde_urlencoded" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.120", + "serde 1.0.136", ] [[package]] name = "sha-1" -version = "0.8.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", ] [[package]] name = "sha-1" -version = "0.9.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest 0.10.3", ] [[package]] @@ -1938,10 +1859,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.0", + "opaque-debug", ] [[package]] @@ -1952,35 +1873,24 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" - -[[package]] -name = "socket2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "winapi", -] +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "socket2" @@ -2035,7 +1945,7 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", - "serde 1.0.120", + "serde 1.0.136", "serde_derive", "syn", ] @@ -2049,7 +1959,7 @@ dependencies = [ "base-x", "proc-macro2", "quote", - "serde 1.0.120", + "serde 1.0.136", "serde_derive", "serde_json", "sha1", @@ -2080,9 +1990,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.67" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6498a9efc342871f91cc2d0d694c674368b4ceb40f62b65a7a08c3792935e702" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" dependencies = [ "proc-macro2", "quote", @@ -2097,13 +2007,13 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -2129,18 +2039,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2", "quote", @@ -2198,9 +2108,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.3.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" dependencies = [ "tinyvec_macros", ] @@ -2213,29 +2123,29 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.7.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" dependencies = [ - "autocfg", "bytes", "libc", "memchr", "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.0", "pin-project-lite", "signal-hook-registry", + "socket2", "tokio-macros", "winapi", ] [[package]] name = "tokio-macros" -version = "1.1.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ "proc-macro2", "quote", @@ -2255,9 +2165,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.2" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76066865172052eb8796c686f0b441a93df8b08d40a950b062ffb9a426f00edd" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", "pin-project-lite", @@ -2266,9 +2176,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a5f475f1b9d077ea1017ecbc60890fda8e54942d680ca0b1d2b47cfa2d861b" +checksum = "511de3f85caf1c98983545490c3d09685fa8eb634e57eec22bb4db271f46cbd8" dependencies = [ "futures-util", "log", @@ -2279,9 +2189,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.7" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes", "futures-core", @@ -2291,13 +2201,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" dependencies = [ - "serde 1.0.120", + "serde 1.0.136", ] [[package]] @@ -2319,8 +2243,8 @@ dependencies = [ "r2d2", "r2d2_mysql", "r2d2_sqlite", - "rand 0.8.4", - "serde 1.0.120", + "rand", + "serde 1.0.136", "serde_bencode", "serde_bytes", "serde_json", @@ -2332,27 +2256,39 @@ dependencies = [ [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "80b9fa4360528139bc96100c160b7ae879f5567f49f1782b0b02035b0358ebf3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "6dfce9f3241b150f36e8e54bb561a742d5daa1a47b5dd9a5ce369fd4a4db2210" dependencies = [ "lazy_static", ] @@ -2365,19 +2301,19 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "tungstenite" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ada8297e8d70872fa9a551d93250a9f407beb9f37ef86494eb20012a2ff7c24" +checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5" dependencies = [ "base64", "byteorder", "bytes", "http", "httparse", - "input_buffer", "log", - "rand 0.8.4", + "rand", "sha-1 0.9.8", + "thiserror", "url", "utf-8", ] @@ -2397,16 +2333,16 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.4", + "cfg-if", + "rand", "static_assertions", ] [[package]] name = "typenum" -version = "1.12.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "unicase" @@ -2419,9 +2355,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" [[package]] name = "unicode-normalization" @@ -2440,9 +2376,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "untrusted" @@ -2488,9 +2424,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" @@ -2504,12 +2440,13 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332d47745e9a0c38636dbd454729b147d16bd1ed08ae67b3ab281c4506771054" +checksum = "3cef4e1e9114a4b7f1ac799f16ce71c14de5778500c5450ec6b7b920c55b587e" dependencies = [ "bytes", - "futures", + "futures-channel", + "futures-util", "headers", "http", "hyper", @@ -2520,45 +2457,45 @@ dependencies = [ "percent-encoding", "pin-project", "scoped-tls", - "serde 1.0.120", + "serde 1.0.136", "serde_json", "serde_urlencoded", "tokio", "tokio-rustls", "tokio-stream", "tokio-tungstenite", - "tokio-util", + "tokio-util 0.6.9", "tower-service", "tracing", ] [[package]] name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.78" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.78" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -2571,9 +2508,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.78" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2581,9 +2518,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.78" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -2594,15 +2531,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.78" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "web-sys" -version = "0.3.55" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -2658,6 +2595,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" + +[[package]] +name = "windows_i686_gnu" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" + +[[package]] +name = "windows_i686_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" + [[package]] name = "wyz" version = "0.4.0" diff --git a/src/tracker.rs b/src/tracker.rs index 4aaa767be..053119b4a 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -34,13 +34,15 @@ pub enum TrackerMode { PrivateListedMode, } + pub struct TorrentTracker { pub config: Arc, torrents: tokio::sync::RwLock>, updates: tokio::sync::RwLock>, shadow: tokio::sync::RwLock>, database: Box, - pub stats_tracker: StatsTracker + pub stats_tracker: StatsTracker, + pub guard: pprof } impl TorrentTracker { @@ -56,7 +58,8 @@ impl TorrentTracker { updates: RwLock::new(std::collections::HashMap::new()), shadow: RwLock::new(std::collections::HashMap::new()), database, - stats_tracker + stats_tracker, + guard: pprof::ProfilerGuard::new(100).unwrap() }) } @@ -228,15 +231,15 @@ impl TorrentTracker { pub async fn post_log(&self) { let torrents = self.torrents.read().await; - let torrents_size = mem::size_of_val(&*torrents); + let torrents_size = torrents.len(); drop(torrents); let updates = self.updates.read().await; - let updates_size = mem::size_of_val(&*updates); + let updates_size = updates.len(); drop(updates); let shadow = self.shadow.read().await; - let shadow_size = mem::size_of_val(&*shadow); + let shadow_size = shadow.len(); drop(shadow); - info!("Stats [::] Torrents: {} byte(s) | Updates: {} byte(s) | Shadow: {} byte(s)", torrents_size, updates_size, shadow_size); + info!("-=[ Stats ]=- | Torrents: {} | Updates: {} | Shadow: {}", torrents_size, updates_size, shadow_size); } // remove torrents without peers if enabled, and defragment memory From 97c17d4a5bb784c43188def982a58c3ad63c5b60 Mon Sep 17 00:00:00 2001 From: Power2All Date: Wed, 13 Apr 2022 19:46:06 +0200 Subject: [PATCH 14/53] Oops :) --- src/tracker.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tracker.rs b/src/tracker.rs index 053119b4a..defdecc1f 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -41,8 +41,7 @@ pub struct TorrentTracker { updates: tokio::sync::RwLock>, shadow: tokio::sync::RwLock>, database: Box, - pub stats_tracker: StatsTracker, - pub guard: pprof + pub stats_tracker: StatsTracker } impl TorrentTracker { @@ -58,8 +57,7 @@ impl TorrentTracker { updates: RwLock::new(std::collections::HashMap::new()), shadow: RwLock::new(std::collections::HashMap::new()), database, - stats_tracker, - guard: pprof::ProfilerGuard::new(100).unwrap() + stats_tracker }) } From 145780a6b770431eec99d3eeeb7dfce247747e3b Mon Sep 17 00:00:00 2001 From: Power2All Date: Fri, 15 Apr 2022 08:57:25 +0200 Subject: [PATCH 15/53] Adding profiling and updates and cleanups --- Cargo.lock | 91 ++++++++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 5 +++ src/main.rs | 7 ++++ src/tracker.rs | 2 -- 4 files changed, 96 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bc7233c8..eb04ba651 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -86,6 +95,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide 0.4.4", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.8" @@ -375,6 +399,21 @@ dependencies = [ "syn", ] +[[package]] +name = "dhat" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47003dc9f6368a88e85956c3b2573a7e6872746a3e5d762a8885da3a136a0381" +dependencies = [ + "backtrace", + "lazy_static", + "parking_lot 0.11.2", + "rustc-hash", + "serde 1.0.136", + "serde_json", + "thousands", +] + [[package]] name = "digest" version = "0.9.0" @@ -459,7 +498,7 @@ dependencies = [ "crc32fast", "libc", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.5.1", ] [[package]] @@ -673,6 +712,12 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" + [[package]] name = "glob" version = "0.3.0" @@ -1020,6 +1065,16 @@ dependencies = [ "unicase", ] +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.5.1" @@ -1253,6 +1308,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.10.0" @@ -1593,6 +1657,12 @@ dependencies = [ "serde 1.0.136", ] +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2057,6 +2127,12 @@ dependencies = [ "syn", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "time" version = "0.1.44" @@ -2217,9 +2293,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde 1.0.136", ] @@ -2235,6 +2311,7 @@ dependencies = [ "chrono", "config", "derive_more", + "dhat", "fern", "futures", "hex", @@ -2262,9 +2339,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80b9fa4360528139bc96100c160b7ae879f5567f49f1782b0b02035b0358ebf3" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if", "log", @@ -2286,9 +2363,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dfce9f3241b150f36e8e54bb561a742d5daa1a47b5dd9a5ce369fd4a4db2210" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", ] diff --git a/Cargo.toml b/Cargo.toml index 81f76abe9..9871f20a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,12 @@ description = "A feature rich BitTorrent tracker." edition = "2018" [profile.release] +debug = 1 lto = "fat" +[features] +dhat-heap = [] # if you are doing heap profiling + [dependencies] serde = {version = "1.0", features = ["derive"]} serde_bencode = "^0.2.3" @@ -34,3 +38,4 @@ thiserror = "1.0" aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } futures = "0.3.21" async-trait = "0.1.52" +dhat = "0.3.0" diff --git a/src/main.rs b/src/main.rs index 9ba45427e..63eb2ec0a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,8 +5,15 @@ use tokio::task::JoinHandle; use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; use torrust_tracker::torrust_http_tracker::server::HttpServer; +#[cfg(feature = "dhat-heap")] +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + #[tokio::main] async fn main() { + #[cfg(feature = "dhat-heap")] + let _profiler = dhat::Profiler::new_heap(); + // torrust config let config = match Configuration::load_from_file() { Ok(config) => Arc::new(config), diff --git a/src/tracker.rs b/src/tracker.rs index defdecc1f..3a69e3e0b 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -6,7 +6,6 @@ use crate::common::{InfoHash}; use std::net::{SocketAddr}; use crate::{Configuration, database, key_manager}; use std::collections::btree_map::Entry; -use std::mem; use std::sync::Arc; use log::info; use crate::key_manager::AuthKey; @@ -34,7 +33,6 @@ pub enum TrackerMode { PrivateListedMode, } - pub struct TorrentTracker { pub config: Arc, torrents: tokio::sync::RwLock>, From 9159187f312bbbc529df662f05d5b2f5bce6fa50 Mon Sep 17 00:00:00 2001 From: Power2All Date: Fri, 15 Apr 2022 14:07:42 +0200 Subject: [PATCH 16/53] First implementation and iteration of using crossbeam to eventually handle the torrent data, should improve and minimize the memory usage drastically, so to keep the memory object at a single place --- Cargo.lock | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/main.rs | 55 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index eb04ba651..00e23092c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -365,6 +365,75 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "crypto-common" version = "0.1.3" @@ -2310,6 +2379,8 @@ dependencies = [ "byteorder", "chrono", "config", + "crossbeam", + "crossbeam-channel", "derive_more", "dhat", "fern", diff --git a/Cargo.toml b/Cargo.toml index 9871f20a4..408a07b85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,3 +39,5 @@ aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } futures = "0.3.21" async-trait = "0.1.52" dhat = "0.3.0" +crossbeam = "0.8.1" +crossbeam-channel = "0.5.4" diff --git a/src/main.rs b/src/main.rs index 63eb2ec0a..2943e4ca7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; use std::sync::Arc; +use crossbeam_channel::bounded; use log::{info}; use tokio::task::JoinHandle; use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; @@ -9,11 +10,28 @@ use torrust_tracker::torrust_http_tracker::server::HttpServer; #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; +pub struct DataStream { + action: u8, + data: Vec<()> +} + #[tokio::main] async fn main() { #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); + // Loading configuration + let config = match Configuration::load_from_file() { + Ok(config) => config, + Err(error) => { + panic!("{}", error) + } + }; + + // Start the thread where data is being exchanged for usaga + let (sender, receiver): (crossbeam_channel::Sender, crossbeam_channel::Receiver) = bounded(1); + let _torrents_memory_handler = start_torrents_memory_handler(&config, sender.clone(), receiver.clone()); + // torrust config let config = match Configuration::load_from_file() { Ok(config) => Arc::new(config), @@ -97,10 +115,47 @@ async fn main() { let _ = tracker.periodic_saving().await; info!("Torrents saved"); } + + // Closing down channel + sender.clone().send(DataStream{ + action: ACTION_CLOSE_CHANNEL, + data: Vec::new() + }); } } } + +const ACTION_CLOSE_CHANNEL: u8 = 0; +const ACTION_READ_TORRENTS: u8 = 1; +const ACTION_WRITE_TORRENTS: u8 = 2; +const ACTION_UPDATE_TORRENTS: u8 = 3; +const ACTION_READ_PEERS: u8 = 4; +const ACTION_WRITE_PEERS: u8 = 5; +const ACTION_UPDATE_PEERS: u8 = 6; +fn start_torrents_memory_handler(config: &Configuration, sender: crossbeam_channel::Sender, receiver: crossbeam_channel::Receiver) -> Option> { + // This is our main memory handler, everything will be received, handled and send back. + return Some(tokio::spawn(async move { + loop { + // Wait for incoming data. + let data: DataStream = receiver.recv().unwrap(); + + // Lets check what action is given. + match data.action { + ACTION_CLOSE_CHANNEL => { + info!("Ending the memory handler thread..."); + sender.send(DataStream{ + action: ACTION_CLOSE_CHANNEL, + data: Vec::new() + }); + break; + } + _ => {} + } + } + })); +} + fn start_torrent_periodic_job(config: Arc, tracker: Arc) -> Option> { let weak_tracker = std::sync::Arc::downgrade(&tracker); let interval = config.persistence_interval.unwrap_or(900); From 03eade11ee9d33dc5e786d9630464dc3c5394ccc Mon Sep 17 00:00:00 2001 From: Power2All Date: Fri, 15 Apr 2022 14:25:54 +0200 Subject: [PATCH 17/53] Moving around, and preparing a refactoring to channel based handling --- src/main.rs | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2943e4ca7..b46fc4c71 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,8 +28,11 @@ async fn main() { } }; + // Enable logging handling + logging::setup_logging(&config); + // Start the thread where data is being exchanged for usaga - let (sender, receiver): (crossbeam_channel::Sender, crossbeam_channel::Receiver) = bounded(1); + let (sender, receiver): (crossbeam_channel::Sender, crossbeam_channel::Receiver) = bounded(0); let _torrents_memory_handler = start_torrents_memory_handler(&config, sender.clone(), receiver.clone()); // torrust config @@ -45,8 +48,6 @@ async fn main() { panic!("{}", e) })); - logging::setup_logging(&config); - // load persistent torrents if enabled if config.persistence { info!("Loading persistent torrents into memory..."); @@ -117,7 +118,7 @@ async fn main() { } // Closing down channel - sender.clone().send(DataStream{ + let _ = sender.clone().send(DataStream{ action: ACTION_CLOSE_CHANNEL, data: Vec::new() }); @@ -135,6 +136,8 @@ const ACTION_WRITE_PEERS: u8 = 5; const ACTION_UPDATE_PEERS: u8 = 6; fn start_torrents_memory_handler(config: &Configuration, sender: crossbeam_channel::Sender, receiver: crossbeam_channel::Receiver) -> Option> { // This is our main memory handler, everything will be received, handled and send back. + info!("Starting memory handler thread..."); + return Some(tokio::spawn(async move { loop { // Wait for incoming data. @@ -144,11 +147,29 @@ fn start_torrents_memory_handler(config: &Configuration, sender: crossbeam_chann match data.action { ACTION_CLOSE_CHANNEL => { info!("Ending the memory handler thread..."); - sender.send(DataStream{ + let _ = sender.send(DataStream{ action: ACTION_CLOSE_CHANNEL, data: Vec::new() }); break; + } + ACTION_READ_TORRENTS => { + + } + ACTION_WRITE_TORRENTS => { + + } + ACTION_UPDATE_TORRENTS => { + + } + ACTION_READ_PEERS => { + + } + ACTION_WRITE_PEERS => { + + } + ACTION_UPDATE_PEERS => { + } _ => {} } From 0a858dc067d6b1f4fbf48e1b8a373655df89d7b9 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sat, 16 Apr 2022 12:48:55 +0200 Subject: [PATCH 18/53] Revert "Moving around, and preparing a refactoring to channel based handling" This reverts commit 03eade11ee9d33dc5e786d9630464dc3c5394ccc. --- src/main.rs | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/src/main.rs b/src/main.rs index b46fc4c71..2943e4ca7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,11 +28,8 @@ async fn main() { } }; - // Enable logging handling - logging::setup_logging(&config); - // Start the thread where data is being exchanged for usaga - let (sender, receiver): (crossbeam_channel::Sender, crossbeam_channel::Receiver) = bounded(0); + let (sender, receiver): (crossbeam_channel::Sender, crossbeam_channel::Receiver) = bounded(1); let _torrents_memory_handler = start_torrents_memory_handler(&config, sender.clone(), receiver.clone()); // torrust config @@ -48,6 +45,8 @@ async fn main() { panic!("{}", e) })); + logging::setup_logging(&config); + // load persistent torrents if enabled if config.persistence { info!("Loading persistent torrents into memory..."); @@ -118,7 +117,7 @@ async fn main() { } // Closing down channel - let _ = sender.clone().send(DataStream{ + sender.clone().send(DataStream{ action: ACTION_CLOSE_CHANNEL, data: Vec::new() }); @@ -136,8 +135,6 @@ const ACTION_WRITE_PEERS: u8 = 5; const ACTION_UPDATE_PEERS: u8 = 6; fn start_torrents_memory_handler(config: &Configuration, sender: crossbeam_channel::Sender, receiver: crossbeam_channel::Receiver) -> Option> { // This is our main memory handler, everything will be received, handled and send back. - info!("Starting memory handler thread..."); - return Some(tokio::spawn(async move { loop { // Wait for incoming data. @@ -147,29 +144,11 @@ fn start_torrents_memory_handler(config: &Configuration, sender: crossbeam_chann match data.action { ACTION_CLOSE_CHANNEL => { info!("Ending the memory handler thread..."); - let _ = sender.send(DataStream{ + sender.send(DataStream{ action: ACTION_CLOSE_CHANNEL, data: Vec::new() }); break; - } - ACTION_READ_TORRENTS => { - - } - ACTION_WRITE_TORRENTS => { - - } - ACTION_UPDATE_TORRENTS => { - - } - ACTION_READ_PEERS => { - - } - ACTION_WRITE_PEERS => { - - } - ACTION_UPDATE_PEERS => { - } _ => {} } From bd680c4e10ff3c2c672145a2bed59a98bf7f5d3b Mon Sep 17 00:00:00 2001 From: Power2All Date: Sat, 16 Apr 2022 12:49:00 +0200 Subject: [PATCH 19/53] Revert "First implementation and iteration of using crossbeam to eventually handle the torrent data, should improve and minimize the memory usage drastically, so to keep the memory object at a single place" This reverts commit 9159187f312bbbc529df662f05d5b2f5bce6fa50. --- Cargo.lock | 71 ----------------------------------------------------- Cargo.toml | 2 -- src/main.rs | 55 ----------------------------------------- 3 files changed, 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00e23092c..eb04ba651 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -365,75 +365,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" -dependencies = [ - "cfg-if", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "lazy_static", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" -dependencies = [ - "cfg-if", - "lazy_static", -] - [[package]] name = "crypto-common" version = "0.1.3" @@ -2379,8 +2310,6 @@ dependencies = [ "byteorder", "chrono", "config", - "crossbeam", - "crossbeam-channel", "derive_more", "dhat", "fern", diff --git a/Cargo.toml b/Cargo.toml index 408a07b85..9871f20a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,5 +39,3 @@ aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } futures = "0.3.21" async-trait = "0.1.52" dhat = "0.3.0" -crossbeam = "0.8.1" -crossbeam-channel = "0.5.4" diff --git a/src/main.rs b/src/main.rs index 2943e4ca7..63eb2ec0a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ use std::net::SocketAddr; use std::sync::Arc; -use crossbeam_channel::bounded; use log::{info}; use tokio::task::JoinHandle; use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; @@ -10,28 +9,11 @@ use torrust_tracker::torrust_http_tracker::server::HttpServer; #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; -pub struct DataStream { - action: u8, - data: Vec<()> -} - #[tokio::main] async fn main() { #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); - // Loading configuration - let config = match Configuration::load_from_file() { - Ok(config) => config, - Err(error) => { - panic!("{}", error) - } - }; - - // Start the thread where data is being exchanged for usaga - let (sender, receiver): (crossbeam_channel::Sender, crossbeam_channel::Receiver) = bounded(1); - let _torrents_memory_handler = start_torrents_memory_handler(&config, sender.clone(), receiver.clone()); - // torrust config let config = match Configuration::load_from_file() { Ok(config) => Arc::new(config), @@ -115,47 +97,10 @@ async fn main() { let _ = tracker.periodic_saving().await; info!("Torrents saved"); } - - // Closing down channel - sender.clone().send(DataStream{ - action: ACTION_CLOSE_CHANNEL, - data: Vec::new() - }); } } } - -const ACTION_CLOSE_CHANNEL: u8 = 0; -const ACTION_READ_TORRENTS: u8 = 1; -const ACTION_WRITE_TORRENTS: u8 = 2; -const ACTION_UPDATE_TORRENTS: u8 = 3; -const ACTION_READ_PEERS: u8 = 4; -const ACTION_WRITE_PEERS: u8 = 5; -const ACTION_UPDATE_PEERS: u8 = 6; -fn start_torrents_memory_handler(config: &Configuration, sender: crossbeam_channel::Sender, receiver: crossbeam_channel::Receiver) -> Option> { - // This is our main memory handler, everything will be received, handled and send back. - return Some(tokio::spawn(async move { - loop { - // Wait for incoming data. - let data: DataStream = receiver.recv().unwrap(); - - // Lets check what action is given. - match data.action { - ACTION_CLOSE_CHANNEL => { - info!("Ending the memory handler thread..."); - sender.send(DataStream{ - action: ACTION_CLOSE_CHANNEL, - data: Vec::new() - }); - break; - } - _ => {} - } - } - })); -} - fn start_torrent_periodic_job(config: Arc, tracker: Arc) -> Option> { let weak_tracker = std::sync::Arc::downgrade(&tracker); let interval = config.persistence_interval.unwrap_or(900); From 509d487f8040a5471d7fa835c39df2aeb476e0de Mon Sep 17 00:00:00 2001 From: Power2All Date: Sat, 16 Apr 2022 13:02:20 +0200 Subject: [PATCH 20/53] Code cleanup and fixing MySQL --- Cargo.toml | 8 +++--- src/common.rs | 7 +++-- src/config.rs | 32 +++++++++++---------- src/database.rs | 12 ++++---- src/http_api_server.rs | 29 ++++++++++--------- src/key_manager.rs | 18 ++++++------ src/lib.rs | 14 ++++++---- src/logging.rs | 1 + src/main.rs | 6 ++-- src/mysql_database.rs | 42 +++++++++++++++------------- src/sqlite_database.rs | 42 +++++++++++++++------------- src/torrent.rs | 10 ++++--- src/torrust_http_tracker/errors.rs | 2 +- src/torrust_http_tracker/filters.rs | 26 +++++++++-------- src/torrust_http_tracker/handlers.rs | 12 ++++---- src/torrust_http_tracker/mod.rs | 16 +++++------ src/torrust_http_tracker/request.rs | 6 ++-- src/torrust_http_tracker/response.rs | 9 +++--- src/torrust_http_tracker/routes.rs | 12 ++++---- src/torrust_http_tracker/server.rs | 1 + src/torrust_udp_tracker/handlers.rs | 18 ++++++------ src/torrust_udp_tracker/mod.rs | 10 +++---- src/torrust_udp_tracker/request.rs | 7 +++-- src/torrust_udp_tracker/server.rs | 8 ++++-- src/tracker.rs | 42 +++++++++++++++------------- src/tracker_stats.rs | 9 +++--- src/utils.rs | 5 ++-- 27 files changed, 222 insertions(+), 182 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9871f20a4..a10d548c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,17 +14,17 @@ lto = "fat" dhat-heap = [] # if you are doing heap profiling [dependencies] -serde = {version = "1.0", features = ["derive"]} +serde = { version = "1.0", features = ["derive"] } serde_bencode = "^0.2.3" serde_bytes = "0.11" serde_json = "1.0.72" hex = "0.4.3" percent-encoding = "2.1.0" -warp = {version = "0.3", features = ["tls"]} -tokio = {version = "1.7", features = ["full"]} +warp = { version = "0.3", features = ["tls"] } +tokio = { version = "1.7", features = ["full"] } binascii = "0.1" toml = "0.5" -log = {version = "0.4", features = ["release_max_level_info"]} +log = { version = "0.4", features = ["release_max_level_info"] } fern = "0.6" chrono = "0.4" byteorder = "1" diff --git a/src/common.rs b/src/common.rs index 4d2f5ec71..5d69ed0e1 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,5 +1,5 @@ -use serde::{Deserialize, Serialize}; use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; +use serde::{Deserialize, Serialize}; pub const MAX_SCRAPE_TORRENTS: u8 = 74; pub const AUTH_KEY_LENGTH: usize = 32; @@ -19,7 +19,7 @@ pub enum AnnounceEventDef { Started, Stopped, Completed, - None + None, } #[derive(Serialize, Deserialize)] @@ -135,7 +135,7 @@ impl PeerId { String::from(std::str::from_utf8(bytes_out).unwrap()) } else { "".to_string() - } + }; } } @@ -218,6 +218,7 @@ impl PeerId { } } } + impl Serialize for PeerId { fn serialize(&self, serializer: S) -> Result where diff --git a/src/config.rs b/src/config.rs index 67078d608..7130d8c92 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,19 +1,21 @@ -pub use crate::tracker::TrackerMode; -use serde::{Serialize, Deserialize, Serializer}; use std; use std::collections::HashMap; use std::fs; -use toml; -use std::net::{IpAddr}; +use std::net::IpAddr; use std::path::Path; use std::str::FromStr; -use config::{ConfigError, Config, File}; + +use config::{Config, ConfigError, File}; +use serde::{Deserialize, Serialize, Serializer}; +use toml; + use crate::database::DatabaseDrivers; +pub use crate::tracker::TrackerMode; #[derive(Serialize, Deserialize, PartialEq)] pub enum TrackerServer { UDP, - HTTP + HTTP, } #[derive(Serialize, Deserialize, Debug)] @@ -30,7 +32,7 @@ pub struct HttpTrackerConfig { #[serde(serialize_with = "none_as_empty_string")] pub ssl_cert_path: Option, #[serde(serialize_with = "none_as_empty_string")] - pub ssl_key_path: Option + pub ssl_key_path: Option, } impl HttpTrackerConfig { @@ -113,7 +115,7 @@ impl Configuration { match Self::load(data.as_slice()) { Ok(cfg) => { Ok(cfg) - }, + } Err(e) => Err(ConfigurationError::ParseError(e)), } } @@ -156,21 +158,21 @@ impl Configuration { enabled: true, bind_address: String::from("127.0.0.1:1212"), access_tokens: [(String::from("admin"), String::from("MyAccessToken"))].iter().cloned().collect(), - } + }, }; configuration.udp_trackers.push( - UdpTrackerConfig{ + UdpTrackerConfig { enabled: false, - bind_address: String::from("0.0.0.0:6969") + bind_address: String::from("0.0.0.0:6969"), } ); configuration.http_trackers.push( - HttpTrackerConfig{ + HttpTrackerConfig { enabled: false, bind_address: String::from("0.0.0.0:6969"), ssl_enabled: false, ssl_cert_path: None, - ssl_key_path: None + ssl_key_path: None, } ); configuration @@ -188,7 +190,7 @@ impl Configuration { eprintln!("Creating config file.."); let config = Configuration::default(); let _ = config.save_to_file(); - return Err(ConfigError::Message(format!("Please edit the config.TOML in the root folder and restart the tracker."))) + return Err(ConfigError::Message(format!("Please edit the config.TOML in the root folder and restart the tracker."))); } let torrust_config: Configuration = config.try_into().map_err(|e| ConfigError::Message(format!("Errors while processing config: {}.", e)))?; @@ -196,7 +198,7 @@ impl Configuration { Ok(torrust_config) } - pub fn save_to_file(&self) -> Result<(), ()>{ + pub fn save_to_file(&self) -> Result<(), ()> { let toml_string = toml::to_string(self).expect("Could not encode TOML value"); fs::write("config.toml", toml_string).expect("Could not write to file!"); Ok(()) diff --git a/src/database.rs b/src/database.rs index 18bf41994..a90161e91 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,18 +1,20 @@ use std::collections::BTreeMap; -use crate::{InfoHash}; -use crate::key_manager::AuthKey; -use crate::sqlite_database::SqliteDatabase; + use async_trait::async_trait; use derive_more::{Display, Error}; use log::debug; +use serde::{Deserialize, Serialize}; + +use crate::InfoHash; +use crate::key_manager::AuthKey; use crate::mysql_database::MysqlDatabase; -use serde::{Serialize, Deserialize}; +use crate::sqlite_database::SqliteDatabase; use crate::torrent::TorrentEntry; #[derive(Serialize, Deserialize, Debug)] pub enum DatabaseDrivers { Sqlite3, - MySQL + MySQL, } pub fn connect_database(db_driver: &DatabaseDrivers, db_path: &str) -> Result, r2d2::Error> { diff --git a/src/http_api_server.rs b/src/http_api_server.rs index eff45fc33..89505cb09 100644 --- a/src/http_api_server.rs +++ b/src/http_api_server.rs @@ -1,10 +1,13 @@ -use crate::tracker::{TorrentTracker}; -use serde::{Deserialize, Serialize}; use std::cmp::min; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use warp::{filters, reply, reply::Reply, serve, Filter, Server}; + +use serde::{Deserialize, Serialize}; +use warp::{Filter, filters, reply, reply::Reply, serve, Server}; + use crate::torrent::TorrentPeer; +use crate::tracker::TorrentTracker; + use super::common::*; #[derive(Deserialize, Debug)] @@ -52,7 +55,7 @@ enum ActionStatus<'a> { impl warp::reject::Reject for ActionStatus<'static> {} -fn authenticate(tokens: HashMap) -> impl Filter + Clone { +fn authenticate(tokens: HashMap) -> impl Filter + Clone { #[derive(Deserialize)] struct AuthToken { token: Option, @@ -69,7 +72,7 @@ fn authenticate(tokens: HashMap) -> impl Filter { if !tokens.contains(&token) { - return Err(warp::reject::custom(ActionStatus::Err { reason: "token not valid".into() })) + return Err(warp::reject::custom(ActionStatus::Err { reason: "token not valid".into() })); } Ok(()) @@ -81,7 +84,7 @@ fn authenticate(tokens: HashMap) -> impl Filter) -> Server + Clone + Send + Sync + 'static> { +pub fn build_server(tracker: Arc) -> Server + Clone + Send + Sync + 'static> { // GET /api/torrents?offset=:u32&limit=:u32 // View torrent list let api_torrents = tracker.clone(); @@ -131,7 +134,7 @@ pub fn build_server(tracker: Arc) -> Server| { async move { - let mut results = Stats{ + let mut results = Stats { torrents: 0, seeders: 0, completed: 0, @@ -147,7 +150,7 @@ pub fn build_server(tracker: Arc) -> Server = db @@ -195,7 +198,7 @@ pub fn build_server(tracker: Arc) -> Server) -> Server)| { async move { - match tracker.remove_torrent_from_whitelist(&info_hash).await { - Ok(_) => Ok(warp::reply::json(&ActionStatus::Ok)), - Err(_) => Err(warp::reject::custom(ActionStatus::Err { reason: "failed to remove torrent from whitelist".into() })) - } + match tracker.remove_torrent_from_whitelist(&info_hash).await { + Ok(_) => Ok(warp::reply::json(&ActionStatus::Ok)), + Err(_) => Err(warp::reject::custom(ActionStatus::Err { reason: "failed to remove torrent from whitelist".into() })) + } } }); diff --git a/src/key_manager.rs b/src/key_manager.rs index b1f16f1dc..507402358 100644 --- a/src/key_manager.rs +++ b/src/key_manager.rs @@ -1,10 +1,12 @@ -use super::common::AUTH_KEY_LENGTH; -use crate::utils::current_time; -use rand::{thread_rng, Rng}; +use derive_more::{Display, Error}; +use log::debug; +use rand::{Rng, thread_rng}; use rand::distributions::Alphanumeric; use serde::Serialize; -use log::debug; -use derive_more::{Display, Error}; + +use crate::utils::current_time; + +use super::common::AUTH_KEY_LENGTH; pub fn generate_auth_key(seconds_valid: u64) -> AuthKey { let key: String = thread_rng() @@ -23,8 +25,8 @@ pub fn generate_auth_key(seconds_valid: u64) -> AuthKey { pub fn verify_auth_key(auth_key: &AuthKey) -> Result<(), Error> { let current_time = current_time(); - if auth_key.valid_until.is_none() { return Err(Error::KeyInvalid) } - if auth_key.valid_until.unwrap() < current_time { return Err(Error::KeyExpired) } + if auth_key.valid_until.is_none() { return Err(Error::KeyInvalid); } + if auth_key.valid_until.unwrap() < current_time { return Err(Error::KeyExpired); } Ok(()) } @@ -67,7 +69,7 @@ pub enum Error { #[display(fmt = "Key is invalid.")] KeyInvalid, #[display(fmt = "Key has expired.")] - KeyExpired + KeyExpired, } impl From for Error { diff --git a/src/lib.rs b/src/lib.rs index 3d928aff4..b6cebfc5e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,11 @@ +pub use torrust_http_tracker::server::*; +pub use torrust_udp_tracker::server::*; + +pub use self::common::*; +pub use self::config::*; +pub use self::http_api_server::*; +pub use self::tracker::*; + pub mod config; pub mod tracker; pub mod http_api_server; @@ -13,9 +21,3 @@ pub mod mysql_database; pub mod torrent; pub mod tracker_stats; -pub use self::config::*; -pub use torrust_udp_tracker::server::*; -pub use torrust_http_tracker::server::*; -pub use self::tracker::*; -pub use self::http_api_server::*; -pub use self::common::*; diff --git a/src/logging.rs b/src/logging.rs index 580e35094..c2e77551f 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,4 +1,5 @@ use log::info; + use crate::Configuration; pub fn setup_logging(cfg: &Configuration) { diff --git a/src/main.rs b/src/main.rs index 63eb2ec0a..b17ef14fe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,9 @@ use std::net::SocketAddr; use std::sync::Arc; -use log::{info}; + +use log::info; use tokio::task::JoinHandle; + use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; use torrust_tracker::torrust_http_tracker::server::HttpServer; @@ -12,7 +14,7 @@ static ALLOC: dhat::Alloc = dhat::Alloc; #[tokio::main] async fn main() { #[cfg(feature = "dhat-heap")] - let _profiler = dhat::Profiler::new_heap(); + let _profiler = dhat::Profiler::new_heap(); // torrust config let config = match Configuration::load_from_file() { diff --git a/src/mysql_database.rs b/src/mysql_database.rs index 0597d46aa..be1fe649b 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -1,18 +1,20 @@ use std::collections::BTreeMap; -use crate::{InfoHash, AUTH_KEY_LENGTH, database}; -use log::debug; -use r2d2::{Pool}; -use crate::key_manager::AuthKey; use std::str::FromStr; -use crate::database::Database; + use async_trait::async_trait; +use log::debug; +use r2d2::Pool; use r2d2_mysql::mysql::{Opts, OptsBuilder, params, TxOpts}; use r2d2_mysql::mysql::prelude::Queryable; use r2d2_mysql::MysqlConnectionManager; + +use crate::{AUTH_KEY_LENGTH, database, InfoHash}; +use crate::database::Database; +use crate::key_manager::AuthKey; use crate::torrent::TorrentEntry; pub struct MysqlDatabase { - pool: Pool + pool: Pool, } impl MysqlDatabase { @@ -79,8 +81,8 @@ impl Database for MysqlDatabase { let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; for (info_hash, torrent_entry) in torrents { - let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.exec_drop("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", (info_hash.to_string(), completed.to_string())); + let (_seeders, completed, _leechers) = torrent_entry.get_stats(); + let _ = db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (?, ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())); } let _ = db_transaction.commit(); @@ -93,13 +95,13 @@ impl Database for MysqlDatabase { match conn.exec_first::("SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", params! { info_hash => info_hash }) .map_err(|_| database::Error::QueryReturnedNoRows)? { - Some(info_hash) => { - Ok(InfoHash::from_str(&info_hash).unwrap()) - }, - None => { - Err(database::Error::InvalidQuery) - } + Some(info_hash) => { + Ok(InfoHash::from_str(&info_hash).unwrap()) } + None => { + Err(database::Error::InvalidQuery) + } + } } async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { @@ -110,7 +112,7 @@ impl Database for MysqlDatabase { match conn.exec_drop("INSERT INTO whitelist (info_hash) VALUES (:info_hash_str)", params! { info_hash_str }) { Ok(_) => { Ok(1) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) @@ -126,7 +128,7 @@ impl Database for MysqlDatabase { match conn.exec_drop("DELETE FROM whitelist WHERE info_hash = :info_hash", params! { info_hash }) { Ok(_) => { Ok(1) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) @@ -142,9 +144,9 @@ impl Database for MysqlDatabase { Some((key, valid_until)) => { Ok(AuthKey { key, - valid_until: Some(valid_until as u64) + valid_until: Some(valid_until as u64), }) - }, + } None => { Err(database::Error::InvalidQuery) } @@ -160,7 +162,7 @@ impl Database for MysqlDatabase { match conn.exec_drop("INSERT INTO `keys` (`key`, valid_until) VALUES (:key, :valid_until)", params! { key, valid_until }) { Ok(_) => { Ok(1) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) @@ -174,7 +176,7 @@ impl Database for MysqlDatabase { match conn.exec_drop("DELETE FROM `keys` WHERE key = :key", params! { key }) { Ok(_) => { Ok(1) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index 5facd99d8..fa519ffd0 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -1,17 +1,19 @@ use std::collections::BTreeMap; -use crate::{InfoHash, AUTH_KEY_LENGTH, database}; +use std::str::FromStr; + +use async_trait::async_trait; use log::debug; -use r2d2_sqlite::{SqliteConnectionManager}; -use r2d2::{Pool}; +use r2d2::Pool; +use r2d2_sqlite::SqliteConnectionManager; use r2d2_sqlite::rusqlite::NO_PARAMS; -use crate::key_manager::AuthKey; -use std::str::FromStr; + +use crate::{AUTH_KEY_LENGTH, database, InfoHash}; use crate::database::Database; -use async_trait::async_trait; +use crate::key_manager::AuthKey; use crate::torrent::TorrentEntry; pub struct SqliteDatabase { - pool: Pool + pool: Pool, } impl SqliteDatabase { @@ -68,7 +70,7 @@ impl Database for SqliteDatabase { Ok((info_hash, completed)) })?; - let torrents: Vec<(InfoHash, u32)> = torrent_iter.filter_map(|x| x.ok() ).collect(); + let torrents: Vec<(InfoHash, u32)> = torrent_iter.filter_map(|x| x.ok()).collect(); Ok(torrents) } @@ -79,8 +81,8 @@ impl Database for SqliteDatabase { let db_transaction = conn.transaction()?; for (info_hash, torrent_entry) in torrents { - let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let (_seeders, completed, _leechers) = torrent_entry.get_stats(); + let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); } let _ = db_transaction.commit(); @@ -109,9 +111,9 @@ impl Database for SqliteDatabase { match conn.execute("INSERT INTO whitelist (info_hash) VALUES (?)", &[info_hash.to_string()]) { Ok(updated) => { - if updated > 0 { return Ok(updated) } + if updated > 0 { return Ok(updated); } Err(database::Error::QueryReturnedNoRows) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) @@ -124,9 +126,9 @@ impl Database for SqliteDatabase { match conn.execute("DELETE FROM whitelist WHERE info_hash = ?", &[info_hash.to_string()]) { Ok(updated) => { - if updated > 0 { return Ok(updated) } + if updated > 0 { return Ok(updated); } Err(database::Error::QueryReturnedNoRows) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) @@ -146,7 +148,7 @@ impl Database for SqliteDatabase { Ok(AuthKey { key, - valid_until: Some(valid_until_i64 as u64) + valid_until: Some(valid_until_i64 as u64), }) } else { Err(database::Error::QueryReturnedNoRows) @@ -157,12 +159,12 @@ impl Database for SqliteDatabase { let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; match conn.execute("INSERT INTO keys (key, valid_until) VALUES (?1, ?2)", - &[auth_key.key.to_string(), auth_key.valid_until.unwrap().to_string()] + &[auth_key.key.to_string(), auth_key.valid_until.unwrap().to_string()], ) { Ok(updated) => { - if updated > 0 { return Ok(updated) } + if updated > 0 { return Ok(updated); } Err(database::Error::QueryReturnedNoRows) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) @@ -175,9 +177,9 @@ impl Database for SqliteDatabase { match conn.execute("DELETE FROM keys WHERE key = ?", &[key]) { Ok(updated) => { - if updated > 0 { return Ok(updated) } + if updated > 0 { return Ok(updated); } Err(database::Error::QueryReturnedNoRows) - }, + } Err(e) => { debug!("{:?}", e); Err(database::Error::InvalidQuery) diff --git a/src/torrent.rs b/src/torrent.rs index ef933d224..e2984a490 100644 --- a/src/torrent.rs +++ b/src/torrent.rs @@ -1,10 +1,12 @@ use std::borrow::Cow; use std::net::{IpAddr, SocketAddr}; + use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; + use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; -use crate::torrust_http_tracker::AnnounceRequest; use crate::common::{AnnounceEventDef, NumberOfBytesDef}; +use crate::torrust_http_tracker::AnnounceRequest; #[derive(PartialEq, Eq, Debug, Clone, Serialize)] pub struct TorrentPeer { @@ -33,7 +35,7 @@ impl TorrentPeer { uploaded: announce_request.bytes_uploaded, downloaded: announce_request.bytes_downloaded, left: announce_request.bytes_left, - event: announce_request.event + event: announce_request.event, } } @@ -58,7 +60,7 @@ impl TorrentPeer { uploaded: NumberOfBytes(announce_request.uploaded as i64), downloaded: NumberOfBytes(announce_request.downloaded as i64), left: NumberOfBytes(announce_request.left as i64), - event + event, } } diff --git a/src/torrust_http_tracker/errors.rs b/src/torrust_http_tracker/errors.rs index d8d6c7623..fe0cf26e6 100644 --- a/src/torrust_http_tracker/errors.rs +++ b/src/torrust_http_tracker/errors.rs @@ -1,5 +1,5 @@ -use warp::reject::Reject; use thiserror::Error; +use warp::reject::Reject; #[derive(Error, Debug)] pub enum ServerError { diff --git a/src/torrust_http_tracker/filters.rs b/src/torrust_http_tracker/filters.rs index 61fa20a45..5c4fc9743 100644 --- a/src/torrust_http_tracker/filters.rs +++ b/src/torrust_http_tracker/filters.rs @@ -2,43 +2,45 @@ use std::convert::Infallible; use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; use std::sync::Arc; + use log::debug; use warp::{Filter, reject, Rejection}; + use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId, TorrentTracker}; use crate::key_manager::AuthKey; use crate::torrust_http_tracker::{AnnounceRequest, AnnounceRequestQuery, ScrapeRequest, ServerError, WebResult}; /// Pass Arc along -pub fn with_tracker(tracker: Arc) -> impl Filter,), Error = Infallible> + Clone { +pub fn with_tracker(tracker: Arc) -> impl Filter, ), Error=Infallible> + Clone { warp::any() .map(move || tracker.clone()) } /// Check for infoHash -pub fn with_info_hash() -> impl Filter,), Error = Rejection> + Clone { +pub fn with_info_hash() -> impl Filter, ), Error=Rejection> + Clone { warp::filters::query::raw() .and_then(info_hashes) } /// Check for PeerId -pub fn with_peer_id() -> impl Filter + Clone { +pub fn with_peer_id() -> impl Filter + Clone { warp::filters::query::raw() .and_then(peer_id) } /// Pass Arc along -pub fn with_auth_key() -> impl Filter,), Error = Infallible> + Clone { +pub fn with_auth_key() -> impl Filter, ), Error=Infallible> + Clone { warp::path::param::() .map(|key: String| { AuthKey::from_string(&key) }) .or_else(|_| async { - Ok::<(Option,), Infallible>((None,)) + Ok::<(Option, ), Infallible>((None, )) }) } /// Check for PeerAddress -pub fn with_peer_addr(on_reverse_proxy: bool) -> impl Filter + Clone { +pub fn with_peer_addr(on_reverse_proxy: bool) -> impl Filter + Clone { warp::addr::remote() .and(warp::header::optional::("X-Forwarded-For")) .map(move |remote_addr: Option, x_forwarded_for: Option| { @@ -48,7 +50,7 @@ pub fn with_peer_addr(on_reverse_proxy: bool) -> impl Filter impl Filter + Clone { +pub fn with_announce_request(on_reverse_proxy: bool) -> impl Filter + Clone { warp::filters::query::query::() .and(with_info_hash()) .and(with_peer_id()) @@ -57,7 +59,7 @@ pub fn with_announce_request(on_reverse_proxy: bool) -> impl Filter impl Filter + Clone { +pub fn with_scrape_request(on_reverse_proxy: bool) -> impl Filter + Clone { warp::any() .and(with_info_hash()) .and(with_peer_addr(on_reverse_proxy)) @@ -129,11 +131,11 @@ async fn peer_id(raw_query: String) -> WebResult { /// Get PeerAddress from RemoteAddress or Forwarded async fn peer_addr((on_reverse_proxy, remote_addr, x_forwarded_for): (bool, Option, Option)) -> WebResult { if !on_reverse_proxy && remote_addr.is_none() { - return Err(reject::custom(ServerError::AddressNotFound)) + return Err(reject::custom(ServerError::AddressNotFound)); } if on_reverse_proxy && x_forwarded_for.is_none() { - return Err(reject::custom(ServerError::AddressNotFound)) + return Err(reject::custom(ServerError::AddressNotFound)); } match on_reverse_proxy { @@ -150,7 +152,7 @@ async fn peer_addr((on_reverse_proxy, remote_addr, x_forwarded_for): (bool, Opti debug!("{}", e); Err(reject::custom(ServerError::AddressNotFound)) }) - }, + } false => Ok(remote_addr.unwrap().ip()) } } @@ -166,7 +168,7 @@ async fn announce_request(announce_request_query: AnnounceRequestQuery, info_has port: announce_request_query.port, left: announce_request_query.left.unwrap_or(0), event: announce_request_query.event, - compact: announce_request_query.compact + compact: announce_request_query.compact, }) } diff --git a/src/torrust_http_tracker/handlers.rs b/src/torrust_http_tracker/handlers.rs index 8762faeaf..994b7b765 100644 --- a/src/torrust_http_tracker/handlers.rs +++ b/src/torrust_http_tracker/handlers.rs @@ -2,9 +2,11 @@ use std::collections::HashMap; use std::convert::Infallible; use std::net::IpAddr; use std::sync::Arc; + use log::debug; use warp::{reject, Rejection, Reply}; -use warp::http::{Response}; +use warp::http::Response; + use crate::{InfoHash, TorrentTracker}; use crate::key_manager::AuthKey; use crate::torrent::{TorrentError, TorrentPeer, TorrentStats}; @@ -34,7 +36,7 @@ pub async fn authenticate(info_hash: &InfoHash, auth_key: &Option, trac /// Handle announce request pub async fn handle_announce(announce_request: AnnounceRequest, auth_key: Option, tracker: Arc) -> WebResult { if let Err(e) = authenticate(&announce_request.info_hash, &auth_key, tracker.clone()).await { - return Err(reject::custom(e)) + return Err(reject::custom(e)); } debug!("{:?}", announce_request); @@ -63,7 +65,7 @@ pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option { @@ -94,7 +96,7 @@ fn send_announce_response(announce_request: &AnnounceRequest, torrent_stats: Tor let http_peers: Vec = peers.iter().map(|peer| Peer { peer_id: peer.peer_id.to_string(), ip: peer.peer_addr.ip(), - port: peer.peer_addr.port() + port: peer.peer_addr.port(), }).collect(); let res = AnnounceResponse { @@ -102,7 +104,7 @@ fn send_announce_response(announce_request: &AnnounceRequest, torrent_stats: Tor interval_min, complete: torrent_stats.seeders, incomplete: torrent_stats.leechers, - peers: http_peers + peers: http_peers, }; // check for compact response request diff --git a/src/torrust_http_tracker/mod.rs b/src/torrust_http_tracker/mod.rs index ea6675dce..07d077577 100644 --- a/src/torrust_http_tracker/mod.rs +++ b/src/torrust_http_tracker/mod.rs @@ -1,3 +1,11 @@ +pub use self::errors::*; +pub use self::filters::*; +pub use self::handlers::*; +pub use self::request::*; +pub use self::response::*; +pub use self::routes::*; +pub use self::server::*; + pub mod server; pub mod request; pub mod response; @@ -6,13 +14,5 @@ pub mod routes; pub mod handlers; pub mod filters; -pub use self::server::*; -pub use self::request::*; -pub use self::response::*; -pub use self::errors::*; -pub use self::routes::*; -pub use self::handlers::*; -pub use self::filters::*; - pub type Bytes = u64; pub type WebResult = std::result::Result; diff --git a/src/torrust_http_tracker/request.rs b/src/torrust_http_tracker/request.rs index 0fb316671..487e53a13 100644 --- a/src/torrust_http_tracker/request.rs +++ b/src/torrust_http_tracker/request.rs @@ -1,5 +1,7 @@ -use std::net::{IpAddr}; -use serde::{Deserialize}; +use std::net::IpAddr; + +use serde::Deserialize; + use crate::{InfoHash, PeerId}; use crate::torrust_http_tracker::Bytes; diff --git a/src/torrust_http_tracker/response.rs b/src/torrust_http_tracker/response.rs index af27bc5e9..f57129cde 100644 --- a/src/torrust_http_tracker/response.rs +++ b/src/torrust_http_tracker/response.rs @@ -2,7 +2,8 @@ use std::collections::HashMap; use std::error::Error; use std::io::Write; use std::net::IpAddr; -use serde::{Serialize}; + +use serde::Serialize; #[derive(Serialize)] pub struct Peer { @@ -18,7 +19,7 @@ pub struct AnnounceResponse { //pub tracker_id: String, pub complete: u32, pub incomplete: u32, - pub peers: Vec + pub peers: Vec, } impl AnnounceResponse { @@ -75,7 +76,7 @@ pub struct ScrapeResponseEntry { #[derive(Serialize)] pub struct ScrapeResponse { - pub files: HashMap + pub files: HashMap, } impl ScrapeResponse { @@ -87,7 +88,7 @@ impl ScrapeResponse { #[derive(Serialize)] pub struct ErrorResponse { #[serde(rename = "failure reason")] - pub failure_reason: String + pub failure_reason: String, } impl ErrorResponse { diff --git a/src/torrust_http_tracker/routes.rs b/src/torrust_http_tracker/routes.rs index 4b4de722f..fb6bf5c16 100644 --- a/src/torrust_http_tracker/routes.rs +++ b/src/torrust_http_tracker/routes.rs @@ -1,11 +1,13 @@ use std::convert::Infallible; use std::sync::Arc; + use warp::{Filter, Rejection}; + use crate::TorrentTracker; -use crate::torrust_http_tracker::{handle_announce, send_error, handle_scrape, with_announce_request, with_auth_key, with_scrape_request, with_tracker}; +use crate::torrust_http_tracker::{handle_announce, handle_scrape, send_error, with_announce_request, with_auth_key, with_scrape_request, with_tracker}; /// All routes -pub fn routes(tracker: Arc,) -> impl Filter + Clone { +pub fn routes(tracker: Arc) -> impl Filter + Clone { root(tracker.clone()) .or(announce(tracker.clone())) .or(scrape(tracker.clone())) @@ -13,7 +15,7 @@ pub fn routes(tracker: Arc,) -> impl Filter -fn root(tracker: Arc,) -> impl Filter + Clone { +fn root(tracker: Arc) -> impl Filter + Clone { warp::any() .and(warp::filters::method::get()) .and(with_announce_request(tracker.config.on_reverse_proxy)) @@ -23,7 +25,7 @@ fn root(tracker: Arc,) -> impl Filter -fn announce(tracker: Arc,) -> impl Filter + Clone { +fn announce(tracker: Arc) -> impl Filter + Clone { warp::path::path("announce") .and(warp::filters::method::get()) .and(with_announce_request(tracker.config.on_reverse_proxy)) @@ -33,7 +35,7 @@ fn announce(tracker: Arc,) -> impl Filter -fn scrape(tracker: Arc,) -> impl Filter + Clone { +fn scrape(tracker: Arc) -> impl Filter + Clone { warp::path::path("scrape") .and(warp::filters::method::get()) .and(with_scrape_request(tracker.config.on_reverse_proxy)) diff --git a/src/torrust_http_tracker/server.rs b/src/torrust_http_tracker/server.rs index 69811b3d9..336670030 100644 --- a/src/torrust_http_tracker/server.rs +++ b/src/torrust_http_tracker/server.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; use std::sync::Arc; + use crate::TorrentTracker; use crate::torrust_http_tracker::routes; diff --git a/src/torrust_udp_tracker/handlers.rs b/src/torrust_udp_tracker/handlers.rs index df1a15451..3b8ece647 100644 --- a/src/torrust_udp_tracker/handlers.rs +++ b/src/torrust_udp_tracker/handlers.rs @@ -1,6 +1,8 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; + use aquatic_udp_protocol::{AnnounceInterval, AnnounceRequest, AnnounceResponse, ConnectRequest, ConnectResponse, ErrorResponse, NumberOfDownloads, NumberOfPeers, Port, Request, Response, ResponsePeer, ScrapeRequest, ScrapeResponse, TorrentScrapeStatistics, TransactionId}; + use crate::{InfoHash, MAX_SCRAPE_TORRENTS, TorrentTracker}; use crate::torrent::{TorrentError, TorrentPeer}; use crate::torrust_udp_tracker::errors::ServerError; @@ -103,15 +105,15 @@ pub async fn handle_announce(remote_addr: SocketAddr, announce_request: &Announc leechers: NumberOfPeers(torrent_stats.leechers as i32), seeders: NumberOfPeers(torrent_stats.seeders as i32), peers: peers.iter() - .filter_map(|peer| if let IpAddr::V4(ip) = peer.peer_addr.ip() { + .filter_map(|peer| if let IpAddr::V4(ip) = peer.peer_addr.ip() { Some(ResponsePeer:: { ip_address: ip, - port: Port(peer.peer_addr.port()) + port: Port(peer.peer_addr.port()), }) } else { None } - ).collect() + ).collect(), }) } else { Response::from(AnnounceResponse { @@ -120,15 +122,15 @@ pub async fn handle_announce(remote_addr: SocketAddr, announce_request: &Announc leechers: NumberOfPeers(torrent_stats.leechers as i32), seeders: NumberOfPeers(torrent_stats.seeders as i32), peers: peers.iter() - .filter_map(|peer| if let IpAddr::V6(ip) = peer.peer_addr.ip() { + .filter_map(|peer| if let IpAddr::V6(ip) = peer.peer_addr.ip() { Some(ResponsePeer:: { ip_address: ip, - port: Port(peer.peer_addr.port()) + port: Port(peer.peer_addr.port()), }) } else { None } - ).collect() + ).collect(), }) }; @@ -150,7 +152,7 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra for info_hash in request.info_hashes.iter() { let info_hash = InfoHash(info_hash.0); - if authenticate(&info_hash, tracker.clone()).await.is_err() { continue } + if authenticate(&info_hash, tracker.clone()).await.is_err() { continue; } let scrape_entry = match db.get(&info_hash) { Some(torrent_info) => { @@ -182,7 +184,7 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra Ok(Response::from(ScrapeResponse { transaction_id: request.transaction_id, - torrent_stats + torrent_stats, })) } diff --git a/src/torrust_udp_tracker/mod.rs b/src/torrust_udp_tracker/mod.rs index 6aa5fbce0..25780ba93 100644 --- a/src/torrust_udp_tracker/mod.rs +++ b/src/torrust_udp_tracker/mod.rs @@ -1,13 +1,13 @@ +pub use self::errors::*; +pub use self::handlers::*; +pub use self::request::*; +pub use self::server::*; + pub mod errors; pub mod request; pub mod server; pub mod handlers; -pub use self::errors::*; -pub use self::request::*; -pub use self::server::*; -pub use self::handlers::*; - pub type Bytes = u64; pub type Port = u16; pub type TransactionId = i64; diff --git a/src/torrust_udp_tracker/request.rs b/src/torrust_udp_tracker/request.rs index f3f67fdc1..6531f54b9 100644 --- a/src/torrust_udp_tracker/request.rs +++ b/src/torrust_udp_tracker/request.rs @@ -1,5 +1,6 @@ -use aquatic_udp_protocol::{AnnounceRequest}; -use crate::{InfoHash}; +use aquatic_udp_protocol::AnnounceRequest; + +use crate::InfoHash; // struct AnnounceRequest { // pub connection_id: i64, @@ -25,7 +26,7 @@ impl AnnounceRequestWrapper { pub fn new(announce_request: AnnounceRequest) -> Self { AnnounceRequestWrapper { announce_request: announce_request.clone(), - info_hash: InfoHash(announce_request.info_hash.0) + info_hash: InfoHash(announce_request.info_hash.0), } } } diff --git a/src/torrust_udp_tracker/server.rs b/src/torrust_udp_tracker/server.rs index cae1e5b94..8dc34d85d 100644 --- a/src/torrust_udp_tracker/server.rs +++ b/src/torrust_udp_tracker/server.rs @@ -1,10 +1,12 @@ use std::io::Cursor; -use std::net::{SocketAddr}; +use std::net::SocketAddr; use std::sync::Arc; -use aquatic_udp_protocol::{Response}; + +use aquatic_udp_protocol::Response; use log::{debug, info}; use tokio::net::UdpSocket; -use crate::{TorrentTracker}; + +use crate::TorrentTracker; use crate::torrust_udp_tracker::{handle_packet, MAX_PACKET_SIZE}; pub struct UdpServer { diff --git a/src/tracker.rs b/src/tracker.rs index 3a69e3e0b..4f31256ea 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -1,15 +1,17 @@ +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::sync::Arc; + +use log::info; use serde::{Deserialize, Serialize}; use serde; -use std::collections::BTreeMap; use tokio::sync::{RwLock, RwLockReadGuard}; -use crate::common::{InfoHash}; -use std::net::{SocketAddr}; + use crate::{Configuration, database, key_manager}; -use std::collections::btree_map::Entry; -use std::sync::Arc; -use log::info; +use crate::common::InfoHash; +use crate::database::Database; use crate::key_manager::AuthKey; -use crate::database::{Database}; use crate::key_manager::Error::KeyInvalid; use crate::torrent::{TorrentEntry, TorrentError, TorrentPeer, TorrentStats}; use crate::tracker_stats::{StatsTracker, TrackerStats}; @@ -39,7 +41,7 @@ pub struct TorrentTracker { updates: tokio::sync::RwLock>, shadow: tokio::sync::RwLock>, database: Box, - pub stats_tracker: StatsTracker + pub stats_tracker: StatsTracker, } impl TorrentTracker { @@ -55,7 +57,7 @@ impl TorrentTracker { updates: RwLock::new(std::collections::HashMap::new()), shadow: RwLock::new(std::collections::HashMap::new()), database, - stats_tracker + stats_tracker, }) } @@ -75,7 +77,7 @@ impl TorrentTracker { let auth_key = key_manager::generate_auth_key(seconds_valid); // add key to database - if let Err(error) = self.database.add_key_to_keys(&auth_key).await { return Err(error) } + if let Err(error) = self.database.add_key_to_keys(&auth_key).await { return Err(error); } Ok(auth_key) } @@ -91,18 +93,18 @@ impl TorrentTracker { pub async fn authenticate_request(&self, info_hash: &InfoHash, key: &Option) -> Result<(), TorrentError> { // no authentication needed in public mode - if self.is_public() { return Ok(()) } + if self.is_public() { return Ok(()); } // check if auth_key is set and valid if self.is_private() { match key { Some(key) => { if self.verify_auth_key(key).await.is_err() { - return Err(TorrentError::PeerKeyNotValid) + return Err(TorrentError::PeerKeyNotValid); } } None => { - return Err(TorrentError::PeerNotAuthenticated) + return Err(TorrentError::PeerNotAuthenticated); } } } @@ -110,7 +112,7 @@ impl TorrentTracker { // check if info_hash is whitelisted if self.is_whitelisted() { if self.is_info_hash_whitelisted(info_hash).await == false { - return Err(TorrentError::TorrentNotWhitelisted) + return Err(TorrentError::TorrentNotWhitelisted); } } @@ -155,7 +157,7 @@ impl TorrentTracker { pub async fn get_torrent_peers( &self, info_hash: &InfoHash, - peer_addr: &SocketAddr + peer_addr: &SocketAddr, ) -> Vec { let read_lock = self.torrents.read().await; match read_lock.get(info_hash) { @@ -205,7 +207,7 @@ impl TorrentTracker { let torrent_entry = TorrentEntry { peers: Default::default(), completed, - seeders + seeders, }; torrents.insert(info_hash.clone(), torrent_entry); } @@ -254,10 +256,10 @@ impl TorrentTracker { // Let's iterate through all torrents, and parse. for hash in torrent_hashes.iter() { - let mut torrent = TorrentEntry{ + let mut torrent = TorrentEntry { peers: BTreeMap::new(), completed: 0, - seeders: 0 + seeders: 0, }; let lock = self.torrents.write().await; @@ -307,10 +309,10 @@ impl TorrentTracker { // We get shadow data into local array to be handled. let mut shadow_copy: BTreeMap = BTreeMap::new(); for (infohash, completed) in shadow.iter() { - shadow_copy.insert(*infohash, TorrentEntry{ + shadow_copy.insert(*infohash, TorrentEntry { peers: Default::default(), completed: *completed, - seeders: 0 + seeders: 0, }); } diff --git a/src/tracker_stats.rs b/src/tracker_stats.rs index 1a6a71c2b..0bcd781ba 100644 --- a/src/tracker_stats.rs +++ b/src/tracker_stats.rs @@ -1,6 +1,7 @@ use std::sync::Arc; + use tokio::sync::{mpsc, RwLock, RwLockReadGuard}; -use tokio::sync::mpsc::{Sender}; +use tokio::sync::mpsc::Sender; use tokio::sync::mpsc::error::SendError; const CHANNEL_BUFFER_SIZE: usize = 65_535; @@ -16,7 +17,7 @@ pub enum TrackerStatsEvent { Udp4Scrape, Udp6Connect, Udp6Announce, - Udp6Scrape + Udp6Scrape, } #[derive(Debug)] @@ -56,14 +57,14 @@ impl TrackerStats { pub struct StatsTracker { channel_sender: Option>, - pub stats: Arc> + pub stats: Arc>, } impl StatsTracker { pub fn new() -> Self { Self { channel_sender: None, - stats: Arc::new(RwLock::new(TrackerStats::new())) + stats: Arc::new(RwLock::new(TrackerStats::new())), } } diff --git a/src/utils.rs b/src/utils.rs index e3a8302df..fb2a94513 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,8 +1,9 @@ -use std::net::SocketAddr; -use std::time::SystemTime; use std::error::Error; use std::fmt::Write; use std::io::Cursor; +use std::net::SocketAddr; +use std::time::SystemTime; + use aquatic_udp_protocol::ConnectionId; use byteorder::{BigEndian, ReadBytesExt}; From 0604cb9ad5318adc57e366011a5da32ad8071f80 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sat, 16 Apr 2022 14:07:12 +0200 Subject: [PATCH 21/53] FIxing MySQL support --- src/mysql_database.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index be1fe649b..523a54cf3 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -36,20 +36,20 @@ impl Database for MysqlDatabase { let create_whitelist_table = " CREATE TABLE IF NOT EXISTS whitelist ( id integer PRIMARY KEY AUTO_INCREMENT, - info_hash VARCHAR(20) NOT NULL UNIQUE + info_hash BINARY(20) NOT NULL UNIQUE );".to_string(); let create_torrents_table = " CREATE TABLE IF NOT EXISTS torrents ( id integer PRIMARY KEY AUTO_INCREMENT, - info_hash VARCHAR(20) NOT NULL UNIQUE, + info_hash BINARY(20) NOT NULL UNIQUE, completed INTEGER DEFAULT 0 NOT NULL );".to_string(); let create_keys_table = format!(" CREATE TABLE IF NOT EXISTS `keys` ( `id` INT NOT NULL AUTO_INCREMENT, - `key` VARCHAR({}) NOT NULL, + `key` BINARY({}) NOT NULL, `valid_until` INT(10) NOT NULL, PRIMARY KEY (`id`), UNIQUE (`key`) @@ -67,7 +67,7 @@ impl Database for MysqlDatabase { async fn load_persistent_torrent_data(&self) -> Result, database::Error> { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT info_hash, completed FROM torrents", |(info_hash_string, completed): (String, u32)| { + let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT HEX(info_hash), completed FROM torrents", |(info_hash_string, completed): (String, u32)| { let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); (info_hash, completed) }).map_err(|_| database::Error::QueryReturnedNoRows)?; @@ -82,7 +82,8 @@ impl Database for MysqlDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (?, ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())); + let _ = db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())); + debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = completed", info_hash.to_string(), completed.to_string()); } let _ = db_transaction.commit(); @@ -93,7 +94,7 @@ impl Database for MysqlDatabase { async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - match conn.exec_first::("SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", params! { info_hash => info_hash }) + match conn.exec_first::("SELECT HEX(info_hash) FROM whitelist WHERE info_hash = UNHEX(:info_hash)", params! { info_hash => info_hash }) .map_err(|_| database::Error::QueryReturnedNoRows)? { Some(info_hash) => { Ok(InfoHash::from_str(&info_hash).unwrap()) @@ -109,7 +110,7 @@ impl Database for MysqlDatabase { let info_hash_str = info_hash.to_string(); - match conn.exec_drop("INSERT INTO whitelist (info_hash) VALUES (:info_hash_str)", params! { info_hash_str }) { + match conn.exec_drop("INSERT INTO whitelist (info_hash) VALUES (UNHEX(:info_hash_str))", params! { info_hash_str }) { Ok(_) => { Ok(1) } @@ -125,7 +126,7 @@ impl Database for MysqlDatabase { let info_hash = info_hash.to_string(); - match conn.exec_drop("DELETE FROM whitelist WHERE info_hash = :info_hash", params! { info_hash }) { + match conn.exec_drop("DELETE FROM whitelist WHERE info_hash = UNHEX(:info_hash)", params! { info_hash }) { Ok(_) => { Ok(1) } From 3a839cddb7962a433bba11838cba0e158357ac35 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 13:24:31 +0200 Subject: [PATCH 22/53] Fixing SQL issue and locking problem --- src/mysql_database.rs | 10 +++++++--- src/tracker.rs | 28 ++++++++++++++++++---------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index 523a54cf3..7ecae214a 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -9,7 +9,7 @@ use r2d2_mysql::mysql::prelude::Queryable; use r2d2_mysql::MysqlConnectionManager; use crate::{AUTH_KEY_LENGTH, database, InfoHash}; -use crate::database::Database; +use crate::database::{Database, Error}; use crate::key_manager::AuthKey; use crate::torrent::TorrentEntry; @@ -82,11 +82,15 @@ impl Database for MysqlDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())); + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())).is_err() { + return Err(Error::InvalidQuery); + } debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = completed", info_hash.to_string(), completed.to_string()); } - let _ = db_transaction.commit(); + if db_transaction.commit().is_err() { + return Err(Error::DatabaseError); + }; Ok(()) } diff --git a/src/tracker.rs b/src/tracker.rs index 4f31256ea..a95c35c24 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -243,6 +243,7 @@ impl TorrentTracker { // remove torrents without peers if enabled, and defragment memory pub async fn cleanup_torrents(&self) { info!("Cleaning torrents..."); + let lock = self.torrents.write().await; // First we create a mapping of all the torrent hashes in a vector, and we use this to iterate through the btreemap. @@ -292,23 +293,30 @@ impl TorrentTracker { pub async fn periodic_saving(&self) { // Get a lock for writing - let mut shadow = self.shadow.write().await; + // let mut shadow = self.shadow.write().await; // We will get the data and insert it into the shadow, while clearing updates. let mut updates = self.updates.write().await; - - for (infohash, completed) in updates.iter() { - if shadow.contains_key(infohash) { - shadow.remove(infohash); - } - shadow.insert(*infohash, *completed); + let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); + // let mut torrent_hashes: Vec = Vec::new(); + for (k, completed) in updates.iter() { + updates_cloned.insert(*k, *completed); } updates.clear(); drop(updates); - // We get shadow data into local array to be handled. + let mut shadows = self.shadow.write().await; + for (k, completed) in updates_cloned.iter() { + if shadows.contains_key(k) { + shadows.remove(k); + } + shadows.insert(*k, *completed); + } + drop(updates_cloned); + + // We updated the shadow data from the updates data, let's handle shadow data as expected. let mut shadow_copy: BTreeMap = BTreeMap::new(); - for (infohash, completed) in shadow.iter() { + for (infohash, completed) in shadows.iter() { shadow_copy.insert(*infohash, TorrentEntry { peers: Default::default(), completed: *completed, @@ -317,7 +325,7 @@ impl TorrentTracker { } // Drop the lock - drop(shadow); + drop(shadows); // We will now save the data from the shadow into the database. // This should not put any strain on the server itself, other then the harddisk/ssd. From 93b053793af0a6e59fb25a921d675505885cc6ee Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sun, 17 Apr 2022 13:31:40 +0200 Subject: [PATCH 23/53] feat: added statistics config option --- src/config.rs | 2 ++ src/torrust_http_tracker/handlers.rs | 8 +++--- src/torrust_udp_tracker/handlers.rs | 12 ++++----- src/tracker.rs | 37 +++++++++++++++++----------- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/config.rs b/src/config.rs index 67078d608..b46f29d69 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,6 +56,7 @@ pub struct Configuration { pub mode: TrackerMode, pub db_driver: DatabaseDrivers, pub db_path: String, + pub statistics: bool, pub persistence: bool, pub persistence_interval: Option, pub cleanup_interval: Option, @@ -141,6 +142,7 @@ impl Configuration { mode: TrackerMode::PublicMode, db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), + statistics: true, persistence: false, persistence_interval: Some(900), cleanup_interval: Some(600), diff --git a/src/torrust_http_tracker/handlers.rs b/src/torrust_http_tracker/handlers.rs index 8762faeaf..8e8f2576f 100644 --- a/src/torrust_http_tracker/handlers.rs +++ b/src/torrust_http_tracker/handlers.rs @@ -49,8 +49,8 @@ pub async fn handle_announce(announce_request: AnnounceRequest, auth_key: Option // send stats event match announce_request.peer_addr { - IpAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp4Announce).await; } - IpAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp6Announce).await; } + IpAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp4Announce).await; } + IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp6Announce).await; } } send_announce_response(&announce_request, torrent_stats, peers, announce_interval, tracker.config.announce_interval_min) @@ -82,8 +82,8 @@ pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp4Scrape).await; } - IpAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Tcp6Scrape).await; } + IpAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp4Scrape).await; } + IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp6Scrape).await; } } send_scrape_response(files) diff --git a/src/torrust_udp_tracker/handlers.rs b/src/torrust_udp_tracker/handlers.rs index df1a15451..c94e2e917 100644 --- a/src/torrust_udp_tracker/handlers.rs +++ b/src/torrust_udp_tracker/handlers.rs @@ -75,8 +75,8 @@ pub async fn handle_connect(remote_addr: SocketAddr, request: &ConnectRequest, t // send stats event match remote_addr { - SocketAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp4Connect).await; } - SocketAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp6Connect).await; } + SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp4Connect).await; } + SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp6Connect).await; } } Ok(response) @@ -134,8 +134,8 @@ pub async fn handle_announce(remote_addr: SocketAddr, announce_request: &Announc // send stats event match remote_addr { - SocketAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp4Announce).await; } - SocketAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp6Announce).await; } + SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp4Announce).await; } + SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp6Announce).await; } } Ok(announce_response) @@ -176,8 +176,8 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra // send stats event match remote_addr { - SocketAddr::V4(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp4Scrape).await; } - SocketAddr::V6(_) => { tracker.stats_tracker.send_event(TrackerStatsEvent::Udp6Scrape).await; } + SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp4Scrape).await; } + SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp6Scrape).await; } } Ok(Response::from(ScrapeResponse { diff --git a/src/tracker.rs b/src/tracker.rs index defdecc1f..7a036c1af 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -6,16 +6,16 @@ use crate::common::{InfoHash}; use std::net::{SocketAddr}; use crate::{Configuration, database, key_manager}; use std::collections::btree_map::Entry; -use std::mem; use std::sync::Arc; use log::info; +use tokio::sync::mpsc::error::SendError; use crate::key_manager::AuthKey; use crate::database::{Database}; use crate::key_manager::Error::KeyInvalid; use crate::torrent::{TorrentEntry, TorrentError, TorrentPeer, TorrentStats}; -use crate::tracker_stats::{StatsTracker, TrackerStats}; +use crate::tracker_stats::{StatsTracker, TrackerStats, TrackerStatsEvent}; -#[derive(Serialize, Deserialize, Clone, PartialEq, Debug)] +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Debug)] pub enum TrackerMode { // Will track every new info hash and serve every peer. #[serde(rename = "public")] @@ -36,12 +36,13 @@ pub enum TrackerMode { pub struct TorrentTracker { + mode: TrackerMode, pub config: Arc, - torrents: tokio::sync::RwLock>, - updates: tokio::sync::RwLock>, - shadow: tokio::sync::RwLock>, - database: Box, - pub stats_tracker: StatsTracker + torrents: RwLock>, + updates: RwLock>, + shadow: RwLock>, + stats_tracker: StatsTracker, + database: Box } impl TorrentTracker { @@ -49,28 +50,30 @@ impl TorrentTracker { let database = database::connect_database(&config.db_driver, &config.db_path)?; let mut stats_tracker = StatsTracker::new(); - stats_tracker.run_worker(); + // starts a thread for updating tracker stats + if config.statistics { stats_tracker.run_worker(); } Ok(TorrentTracker { - config, + mode: config.mode, + config: config.clone(), torrents: RwLock::new(std::collections::BTreeMap::new()), updates: RwLock::new(std::collections::HashMap::new()), shadow: RwLock::new(std::collections::HashMap::new()), - database, - stats_tracker + stats_tracker, + database }) } pub fn is_public(&self) -> bool { - self.config.mode == TrackerMode::PublicMode + self.mode == TrackerMode::PublicMode } pub fn is_private(&self) -> bool { - self.config.mode == TrackerMode::PrivateMode || self.config.mode == TrackerMode::PrivateListedMode + self.mode == TrackerMode::PrivateMode || self.mode == TrackerMode::PrivateListedMode } pub fn is_whitelisted(&self) -> bool { - self.config.mode == TrackerMode::ListedMode || self.config.mode == TrackerMode::PrivateListedMode + self.mode == TrackerMode::ListedMode || self.mode == TrackerMode::PrivateListedMode } pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { @@ -227,6 +230,10 @@ impl TorrentTracker { self.stats_tracker.get_stats().await } + pub async fn send_stats_event(&self, event: TrackerStatsEvent) -> Option>> { + self.stats_tracker.send_event(event).await + } + pub async fn post_log(&self) { let torrents = self.torrents.read().await; let torrents_size = torrents.len(); From 9d0714db67db0b17610f525516f2ff5d9570c256 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 14:04:15 +0200 Subject: [PATCH 24/53] Updating README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 929585c11..bb4649271 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Torrust Tracker is a lightweight but incredibly powerful and feature-rich BitTor * [X] Peer authentication using time-bound keys * [X] newTrackon check supported for both HTTP, UDP, where IPv4 and IPv6 is properly handled * [X] SQLite3 Persistent loading and saving of the torrent hashes and completed count +* [X] MySQL support added as engine option +* [X] Periodically saving added, interval can be configured ### Implemented BEPs * [BEP 3](https://www.bittorrent.org/beps/bep_0003.html): The BitTorrent Protocol From 9bc1ebb6057e94b7c70e4571f6e4ab67da6c142c Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 15:50:59 +0200 Subject: [PATCH 25/53] Fixing possible bug --- src/tracker.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tracker.rs b/src/tracker.rs index 0e42f69e1..a67cb62e6 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use log::info; +use log::{debug, info}; use serde::{Deserialize, Serialize}; use serde; use tokio::sync::{RwLock, RwLockReadGuard}; @@ -128,6 +128,7 @@ impl TorrentTracker { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { + debug!("{:#?}", torrent); let _ = self.add_torrent(torrent.0, 0, torrent.1, 0).await; } @@ -308,7 +309,7 @@ impl TorrentTracker { let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); for (k, completed) in updates.iter() { - updates_cloned.insert(*k, *completed); + updates_cloned.insert(k.clone(), completed.clone()); } updates.clear(); drop(updates); @@ -318,16 +319,16 @@ impl TorrentTracker { if shadows.contains_key(k) { shadows.remove(k); } - shadows.insert(*k, *completed); + shadows.insert(k.clone(), completed.clone()); } drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. let mut shadow_copy: BTreeMap = BTreeMap::new(); for (infohash, completed) in shadows.iter() { - shadow_copy.insert(*infohash, TorrentEntry { + shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), - completed: *completed, + completed: completed.clone(), seeders: 0, }); } From 0c332cf382cbc26c325cd79e557b894e3aa57d9c Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 16:24:04 +0200 Subject: [PATCH 26/53] Fixing a buggy SQLite3 and MySQL --- src/mysql_database.rs | 4 ++-- src/sqlite_database.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index 7ecae214a..eea2190a3 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -82,10 +82,10 @@ impl Database for MysqlDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())).is_err() { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", (info_hash.to_string(), completed.to_string())).is_err() { return Err(Error::InvalidQuery); } - debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = completed", info_hash.to_string(), completed.to_string()); + debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", info_hash.to_string(), completed.to_string()); } if db_transaction.commit().is_err() { diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index fa519ffd0..82bb9d4fc 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -82,7 +82,8 @@ impl Database for SqliteDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let _ = db_transaction.execute("INSERT OR IGNORE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let _ = db_transaction.execute("UPDATE torrents SET completed = ? WHERE info_hash = ?", &[completed.to_string(), info_hash.to_string()]); } let _ = db_transaction.commit(); From 84c6ca460c0f8d6a6bb1c63e1203bfe1530439a0 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 17:04:52 +0200 Subject: [PATCH 27/53] Speeding up inserting --- src/mysql_database.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index eea2190a3..ac8bbb54a 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -80,12 +80,24 @@ impl Database for MysqlDatabase { let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; + let mut insert_vector= vec![]; + for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", (info_hash.to_string(), completed.to_string())).is_err() { + insert_vector.push("(UNHEX('" + info_hash.to_string() + "'), " + completed.to_string() + ")"); + if insert_vector.len() == 1000 { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { + return Err(Error::InvalidQuery); + } + insert_vector.clear(); + } + } + + if insert_vector.len() != 0 { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { return Err(Error::InvalidQuery); } - debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", info_hash.to_string(), completed.to_string()); + insert_vector.clear(); } if db_transaction.commit().is_err() { From 8ebd1e5f1164dff111049efcda15a43bd1277a1b Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 17:13:19 +0200 Subject: [PATCH 28/53] Typo's --- src/mysql_database.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index ac8bbb54a..15e2de633 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -84,9 +84,10 @@ impl Database for MysqlDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - insert_vector.push("(UNHEX('" + info_hash.to_string() + "'), " + completed.to_string() + ")"); + insert_vector.push(format!("(UNHEX('{}'), {})", info_hash.to_string(), completed.to_string())); if insert_vector.len() == 1000 { - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { + let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); + if db_transaction.query_drop(query).is_err() { return Err(Error::InvalidQuery); } insert_vector.clear(); @@ -94,7 +95,8 @@ impl Database for MysqlDatabase { } if insert_vector.len() != 0 { - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { + let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); + if db_transaction.query_drop(query).is_err() { return Err(Error::InvalidQuery); } insert_vector.clear(); From 1b4decac685b374cf1b4abab9cae64fb98a21ebc Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 22:06:33 +0200 Subject: [PATCH 29/53] Looking for reason of timeout --- src/tracker.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tracker.rs b/src/tracker.rs index a67cb62e6..c523cefba 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -308,6 +308,7 @@ impl TorrentTracker { let mut updates = self.updates.write().await; let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); + info!("Copying updates to updates_cloned..."); for (k, completed) in updates.iter() { updates_cloned.insert(k.clone(), completed.clone()); } @@ -315,6 +316,7 @@ impl TorrentTracker { drop(updates); let mut shadows = self.shadow.write().await; + info!("Copying updates_cloned into the shadow to overwrite..."); for (k, completed) in updates_cloned.iter() { if shadows.contains_key(k) { shadows.remove(k); @@ -324,6 +326,7 @@ impl TorrentTracker { drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. + info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { @@ -338,11 +341,15 @@ impl TorrentTracker { // We will now save the data from the shadow into the database. // This should not put any strain on the server itself, other then the harddisk/ssd. + info!("Start saving shadow data into SQL..."); let result = self.database.save_persistent_torrent_data(&shadow_copy).await; if result.is_ok() { + info!("Done saving data to SQL and succeeded, emptying shadow..."); let mut shadow = self.shadow.write().await; shadow.clear(); drop(shadow); + } else { + info!("Done saving data to SQL and failed, not emptying shadow..."); } } } From 035c28d9cc6d52c83e08eda87b99b1562db60b75 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 22:10:16 +0200 Subject: [PATCH 30/53] Improving locking --- src/tracker.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tracker.rs b/src/tracker.rs index c523cefba..8ad1faf2f 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -315,19 +315,21 @@ impl TorrentTracker { updates.clear(); drop(updates); - let mut shadows = self.shadow.write().await; info!("Copying updates_cloned into the shadow to overwrite..."); for (k, completed) in updates_cloned.iter() { + let mut shadows = self.shadow.write().await; if shadows.contains_key(k) { shadows.remove(k); } shadows.insert(k.clone(), completed.clone()); + drop(shadows); } drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); + let mut shadows = self.shadow.write().await; for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), @@ -335,8 +337,6 @@ impl TorrentTracker { seeders: 0, }); } - - // Drop the lock drop(shadows); // We will now save the data from the shadow into the database. From aea2de04851058a58faf940f20cff65b315605f4 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 22:12:11 +0200 Subject: [PATCH 31/53] Typo --- src/tracker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracker.rs b/src/tracker.rs index 8ad1faf2f..c0c25bc41 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -329,7 +329,7 @@ impl TorrentTracker { // We updated the shadow data from the updates data, let's handle shadow data as expected. info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); - let mut shadows = self.shadow.write().await; + let shadows = self.shadow.read().await; for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), From 5d1ad9d959ad3de5cf3bac43142266a8a12ab418 Mon Sep 17 00:00:00 2001 From: WarmBeer Date: Mon, 25 Apr 2022 19:13:48 +0200 Subject: [PATCH 32/53] Revert "Development" --- README.md | 2 -- src/mysql_database.rs | 18 ++---------------- src/sqlite_database.rs | 3 +-- src/tracker.rs | 24 ++++++++---------------- 4 files changed, 11 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index bb4649271..929585c11 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,6 @@ Torrust Tracker is a lightweight but incredibly powerful and feature-rich BitTor * [X] Peer authentication using time-bound keys * [X] newTrackon check supported for both HTTP, UDP, where IPv4 and IPv6 is properly handled * [X] SQLite3 Persistent loading and saving of the torrent hashes and completed count -* [X] MySQL support added as engine option -* [X] Periodically saving added, interval can be configured ### Implemented BEPs * [BEP 3](https://www.bittorrent.org/beps/bep_0003.html): The BitTorrent Protocol diff --git a/src/mysql_database.rs b/src/mysql_database.rs index 15e2de633..7ecae214a 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -80,26 +80,12 @@ impl Database for MysqlDatabase { let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; - let mut insert_vector= vec![]; - for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - insert_vector.push(format!("(UNHEX('{}'), {})", info_hash.to_string(), completed.to_string())); - if insert_vector.len() == 1000 { - let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); - if db_transaction.query_drop(query).is_err() { - return Err(Error::InvalidQuery); - } - insert_vector.clear(); - } - } - - if insert_vector.len() != 0 { - let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); - if db_transaction.query_drop(query).is_err() { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())).is_err() { return Err(Error::InvalidQuery); } - insert_vector.clear(); + debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = completed", info_hash.to_string(), completed.to_string()); } if db_transaction.commit().is_err() { diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index 82bb9d4fc..fa519ffd0 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -82,8 +82,7 @@ impl Database for SqliteDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR IGNORE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); - let _ = db_transaction.execute("UPDATE torrents SET completed = ? WHERE info_hash = ?", &[completed.to_string(), info_hash.to_string()]); + let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); } let _ = db_transaction.commit(); diff --git a/src/tracker.rs b/src/tracker.rs index c0c25bc41..0e42f69e1 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use log::{debug, info}; +use log::info; use serde::{Deserialize, Serialize}; use serde; use tokio::sync::{RwLock, RwLockReadGuard}; @@ -128,7 +128,6 @@ impl TorrentTracker { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { - debug!("{:#?}", torrent); let _ = self.add_torrent(torrent.0, 0, torrent.1, 0).await; } @@ -308,48 +307,41 @@ impl TorrentTracker { let mut updates = self.updates.write().await; let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); - info!("Copying updates to updates_cloned..."); for (k, completed) in updates.iter() { - updates_cloned.insert(k.clone(), completed.clone()); + updates_cloned.insert(*k, *completed); } updates.clear(); drop(updates); - info!("Copying updates_cloned into the shadow to overwrite..."); + let mut shadows = self.shadow.write().await; for (k, completed) in updates_cloned.iter() { - let mut shadows = self.shadow.write().await; if shadows.contains_key(k) { shadows.remove(k); } - shadows.insert(k.clone(), completed.clone()); - drop(shadows); + shadows.insert(*k, *completed); } drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. - info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); - let shadows = self.shadow.read().await; for (infohash, completed) in shadows.iter() { - shadow_copy.insert(infohash.clone(), TorrentEntry { + shadow_copy.insert(*infohash, TorrentEntry { peers: Default::default(), - completed: completed.clone(), + completed: *completed, seeders: 0, }); } + + // Drop the lock drop(shadows); // We will now save the data from the shadow into the database. // This should not put any strain on the server itself, other then the harddisk/ssd. - info!("Start saving shadow data into SQL..."); let result = self.database.save_persistent_torrent_data(&shadow_copy).await; if result.is_ok() { - info!("Done saving data to SQL and succeeded, emptying shadow..."); let mut shadow = self.shadow.write().await; shadow.clear(); drop(shadow); - } else { - info!("Done saving data to SQL and failed, not emptying shadow..."); } } } From cef2016c918825694ea51eaa4b4527c62af454f9 Mon Sep 17 00:00:00 2001 From: WarmBeer Date: Mon, 25 Apr 2022 20:06:29 +0200 Subject: [PATCH 33/53] Revert "Revert "Development"" --- README.md | 2 ++ src/mysql_database.rs | 18 ++++++++++++++++-- src/sqlite_database.rs | 3 ++- src/tracker.rs | 24 ++++++++++++++++-------- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 929585c11..bb4649271 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Torrust Tracker is a lightweight but incredibly powerful and feature-rich BitTor * [X] Peer authentication using time-bound keys * [X] newTrackon check supported for both HTTP, UDP, where IPv4 and IPv6 is properly handled * [X] SQLite3 Persistent loading and saving of the torrent hashes and completed count +* [X] MySQL support added as engine option +* [X] Periodically saving added, interval can be configured ### Implemented BEPs * [BEP 3](https://www.bittorrent.org/beps/bep_0003.html): The BitTorrent Protocol diff --git a/src/mysql_database.rs b/src/mysql_database.rs index 7ecae214a..15e2de633 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -80,12 +80,26 @@ impl Database for MysqlDatabase { let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; + let mut insert_vector= vec![]; + for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())).is_err() { + insert_vector.push(format!("(UNHEX('{}'), {})", info_hash.to_string(), completed.to_string())); + if insert_vector.len() == 1000 { + let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); + if db_transaction.query_drop(query).is_err() { + return Err(Error::InvalidQuery); + } + insert_vector.clear(); + } + } + + if insert_vector.len() != 0 { + let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); + if db_transaction.query_drop(query).is_err() { return Err(Error::InvalidQuery); } - debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = completed", info_hash.to_string(), completed.to_string()); + insert_vector.clear(); } if db_transaction.commit().is_err() { diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index fa519ffd0..82bb9d4fc 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -82,7 +82,8 @@ impl Database for SqliteDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let _ = db_transaction.execute("INSERT OR IGNORE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let _ = db_transaction.execute("UPDATE torrents SET completed = ? WHERE info_hash = ?", &[completed.to_string(), info_hash.to_string()]); } let _ = db_transaction.commit(); diff --git a/src/tracker.rs b/src/tracker.rs index 0e42f69e1..c0c25bc41 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use log::info; +use log::{debug, info}; use serde::{Deserialize, Serialize}; use serde; use tokio::sync::{RwLock, RwLockReadGuard}; @@ -128,6 +128,7 @@ impl TorrentTracker { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { + debug!("{:#?}", torrent); let _ = self.add_torrent(torrent.0, 0, torrent.1, 0).await; } @@ -307,41 +308,48 @@ impl TorrentTracker { let mut updates = self.updates.write().await; let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); + info!("Copying updates to updates_cloned..."); for (k, completed) in updates.iter() { - updates_cloned.insert(*k, *completed); + updates_cloned.insert(k.clone(), completed.clone()); } updates.clear(); drop(updates); - let mut shadows = self.shadow.write().await; + info!("Copying updates_cloned into the shadow to overwrite..."); for (k, completed) in updates_cloned.iter() { + let mut shadows = self.shadow.write().await; if shadows.contains_key(k) { shadows.remove(k); } - shadows.insert(*k, *completed); + shadows.insert(k.clone(), completed.clone()); + drop(shadows); } drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. + info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); + let shadows = self.shadow.read().await; for (infohash, completed) in shadows.iter() { - shadow_copy.insert(*infohash, TorrentEntry { + shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), - completed: *completed, + completed: completed.clone(), seeders: 0, }); } - - // Drop the lock drop(shadows); // We will now save the data from the shadow into the database. // This should not put any strain on the server itself, other then the harddisk/ssd. + info!("Start saving shadow data into SQL..."); let result = self.database.save_persistent_torrent_data(&shadow_copy).await; if result.is_ok() { + info!("Done saving data to SQL and succeeded, emptying shadow..."); let mut shadow = self.shadow.write().await; shadow.clear(); drop(shadow); + } else { + info!("Done saving data to SQL and failed, not emptying shadow..."); } } } From 6578c3545a4254c505d92e568a3a5e7cc0b57016 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 2 May 2022 12:42:18 +0200 Subject: [PATCH 34/53] chore: removed dhat --- Cargo.lock | 81 ++--------------------------------------------------- Cargo.toml | 4 --- src/main.rs | 32 +++++++++------------ 3 files changed, 15 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb04ba651..56ecd77a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" -dependencies = [ - "gimli", -] - [[package]] name = "adler" version = "1.0.2" @@ -95,21 +86,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "backtrace" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide 0.4.4", - "object", - "rustc-demangle", -] - [[package]] name = "base-x" version = "0.2.8" @@ -399,21 +375,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dhat" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47003dc9f6368a88e85956c3b2573a7e6872746a3e5d762a8885da3a136a0381" -dependencies = [ - "backtrace", - "lazy_static", - "parking_lot 0.11.2", - "rustc-hash", - "serde 1.0.136", - "serde_json", - "thousands", -] - [[package]] name = "digest" version = "0.9.0" @@ -498,7 +459,7 @@ dependencies = [ "crc32fast", "libc", "libz-sys", - "miniz_oxide 0.5.1", + "miniz_oxide", ] [[package]] @@ -712,12 +673,6 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] -[[package]] -name = "gimli" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" - [[package]] name = "glob" version = "0.3.0" @@ -1065,16 +1020,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "miniz_oxide" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg", -] - [[package]] name = "miniz_oxide" version = "0.5.1" @@ -1308,15 +1253,6 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.10.0" @@ -1657,12 +1593,6 @@ dependencies = [ "serde 1.0.136", ] -[[package]] -name = "rustc-demangle" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -2127,12 +2057,6 @@ dependencies = [ "syn", ] -[[package]] -name = "thousands" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" - [[package]] name = "time" version = "0.1.44" @@ -2302,7 +2226,7 @@ dependencies = [ [[package]] name = "torrust-tracker" -version = "2.2.1" +version = "2.3.0" dependencies = [ "aquatic_udp_protocol", "async-trait", @@ -2311,7 +2235,6 @@ dependencies = [ "chrono", "config", "derive_more", - "dhat", "fern", "futures", "hex", diff --git a/Cargo.toml b/Cargo.toml index a10d548c2..7be32ce89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,9 +10,6 @@ edition = "2018" debug = 1 lto = "fat" -[features] -dhat-heap = [] # if you are doing heap profiling - [dependencies] serde = { version = "1.0", features = ["derive"] } serde_bencode = "^0.2.3" @@ -38,4 +35,3 @@ thiserror = "1.0" aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } futures = "0.3.21" async-trait = "0.1.52" -dhat = "0.3.0" diff --git a/src/main.rs b/src/main.rs index b17ef14fe..5e16f2e0c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,15 +7,8 @@ use tokio::task::JoinHandle; use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; use torrust_tracker::torrust_http_tracker::server::HttpServer; -#[cfg(feature = "dhat-heap")] -#[global_allocator] -static ALLOC: dhat::Alloc = dhat::Alloc; - #[tokio::main] async fn main() { - #[cfg(feature = "dhat-heap")] - let _profiler = dhat::Profiler::new_heap(); - // torrust config let config = match Configuration::load_from_file() { Ok(config) => Arc::new(config), @@ -29,6 +22,7 @@ async fn main() { panic!("{}", e) })); + // initialize logging logging::setup_logging(&config); // load persistent torrents if enabled @@ -39,11 +33,11 @@ async fn main() { }; info!("Persistent torrents loaded."); - let _torrent_periodic_job = start_torrent_periodic_job(config.clone(), tracker.clone()).unwrap(); + let _torrent_periodic_job = start_torrent_periodic_job(config.clone(), tracker.clone()); } // start torrent cleanup job (periodically removes old peers) - let _torrent_cleanup_job = start_torrent_cleanup_job(config.clone(), tracker.clone()).unwrap(); + let _torrent_cleanup_job = start_torrent_cleanup_job(config.clone(), tracker.clone()); // start HTTP API server if config.http_api.enabled { @@ -80,7 +74,7 @@ async fn main() { } // start a thread to post statistics - let _ = start_statistics_job(config.clone(), tracker.clone()).unwrap(); + let _ = start_statistics_job(config.clone(), tracker.clone()); // handle the signals here tokio::select! { @@ -103,11 +97,11 @@ async fn main() { } } -fn start_torrent_periodic_job(config: Arc, tracker: Arc) -> Option> { +fn start_torrent_periodic_job(config: Arc, tracker: Arc) -> JoinHandle<()> { let weak_tracker = std::sync::Arc::downgrade(&tracker); let interval = config.persistence_interval.unwrap_or(900); - return Some(tokio::spawn(async move { + tokio::spawn(async move { let interval = std::time::Duration::from_secs(interval); let mut interval = tokio::time::interval(interval); interval.tick().await; // first tick is immediate... @@ -122,14 +116,14 @@ fn start_torrent_periodic_job(config: Arc, tracker: Arc, tracker: Arc) -> Option> { +fn start_torrent_cleanup_job(config: Arc, tracker: Arc) -> JoinHandle<()> { let weak_tracker = std::sync::Arc::downgrade(&tracker); let interval = config.cleanup_interval.unwrap_or(600); - return Some(tokio::spawn(async move { + tokio::spawn(async move { let interval = std::time::Duration::from_secs(interval); let mut interval = tokio::time::interval(interval); interval.tick().await; // first tick is immediate... @@ -142,14 +136,14 @@ fn start_torrent_cleanup_job(config: Arc, tracker: Arc, tracker: Arc) -> Option> { +fn start_statistics_job(config: Arc, tracker: Arc) -> JoinHandle<()> { let weak_tracker = std::sync::Arc::downgrade(&tracker); let interval = config.log_interval.unwrap_or(60); - return Some(tokio::spawn(async move { + tokio::spawn(async move { let interval = std::time::Duration::from_secs(interval); let mut interval = tokio::time::interval(interval); interval.tick().await; // first tick is immediate... @@ -162,7 +156,7 @@ fn start_statistics_job(config: Arc, tracker: Arc break; } } - })); + }) } fn start_api_server(config: &HttpApiConfig, tracker: Arc) -> JoinHandle<()> { From 52785a0b58097151eb85c663269f9210fb51cea5 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 2 May 2022 23:04:37 +0200 Subject: [PATCH 35/53] refactor: renamed http and udp modules --- Cargo.toml | 2 +- src/{torrust_http_tracker => http}/errors.rs | 0 src/{torrust_http_tracker => http}/filters.rs | 2 +- src/{torrust_http_tracker => http}/handlers.rs | 2 +- src/{torrust_http_tracker => http}/mod.rs | 0 src/{torrust_http_tracker => http}/request.rs | 2 +- src/{torrust_http_tracker => http}/response.rs | 0 src/{torrust_http_tracker => http}/routes.rs | 2 +- src/{torrust_http_tracker => http}/server.rs | 2 +- src/lib.rs | 8 ++++---- src/main.rs | 2 +- src/torrent.rs | 2 +- src/{torrust_udp_tracker => udp}/errors.rs | 0 src/{torrust_udp_tracker => udp}/handlers.rs | 4 ++-- src/{torrust_udp_tracker => udp}/mod.rs | 0 src/{torrust_udp_tracker => udp}/request.rs | 0 src/{torrust_udp_tracker => udp}/server.rs | 2 +- 17 files changed, 15 insertions(+), 15 deletions(-) rename src/{torrust_http_tracker => http}/errors.rs (100%) rename src/{torrust_http_tracker => http}/filters.rs (98%) rename src/{torrust_http_tracker => http}/handlers.rs (96%) rename src/{torrust_http_tracker => http}/mod.rs (100%) rename src/{torrust_http_tracker => http}/request.rs (94%) rename src/{torrust_http_tracker => http}/response.rs (100%) rename src/{torrust_http_tracker => http}/routes.rs (90%) rename src/{torrust_http_tracker => http}/server.rs (97%) rename src/{torrust_udp_tracker => udp}/errors.rs (100%) rename src/{torrust_udp_tracker => udp}/handlers.rs (98%) rename src/{torrust_udp_tracker => udp}/mod.rs (100%) rename src/{torrust_udp_tracker => udp}/request.rs (100%) rename src/{torrust_udp_tracker => udp}/server.rs (97%) diff --git a/Cargo.toml b/Cargo.toml index 7be32ce89..cc97072a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "torrust-tracker" -version = "2.2.1" +version = "2.3.0" license = "AGPL-3.0" authors = ["Mick van Dijke "] description = "A feature rich BitTorrent tracker." diff --git a/src/torrust_http_tracker/errors.rs b/src/http/errors.rs similarity index 100% rename from src/torrust_http_tracker/errors.rs rename to src/http/errors.rs diff --git a/src/torrust_http_tracker/filters.rs b/src/http/filters.rs similarity index 98% rename from src/torrust_http_tracker/filters.rs rename to src/http/filters.rs index 5c4fc9743..5e0c3e068 100644 --- a/src/torrust_http_tracker/filters.rs +++ b/src/http/filters.rs @@ -8,7 +8,7 @@ use warp::{Filter, reject, Rejection}; use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId, TorrentTracker}; use crate::key_manager::AuthKey; -use crate::torrust_http_tracker::{AnnounceRequest, AnnounceRequestQuery, ScrapeRequest, ServerError, WebResult}; +use crate::http::{AnnounceRequest, AnnounceRequestQuery, ScrapeRequest, ServerError, WebResult}; /// Pass Arc along pub fn with_tracker(tracker: Arc) -> impl Filter, ), Error=Infallible> + Clone { diff --git a/src/torrust_http_tracker/handlers.rs b/src/http/handlers.rs similarity index 96% rename from src/torrust_http_tracker/handlers.rs rename to src/http/handlers.rs index 9021e8858..c81c93d9b 100644 --- a/src/torrust_http_tracker/handlers.rs +++ b/src/http/handlers.rs @@ -10,7 +10,7 @@ use warp::http::Response; use crate::{InfoHash, TorrentTracker}; use crate::key_manager::AuthKey; use crate::torrent::{TorrentError, TorrentPeer, TorrentStats}; -use crate::torrust_http_tracker::{AnnounceRequest, AnnounceResponse, ErrorResponse, Peer, ScrapeRequest, ScrapeResponse, ScrapeResponseEntry, ServerError, WebResult}; +use crate::http::{AnnounceRequest, AnnounceResponse, ErrorResponse, Peer, ScrapeRequest, ScrapeResponse, ScrapeResponseEntry, ServerError, WebResult}; use crate::tracker_stats::TrackerStatsEvent; use crate::utils::url_encode_bytes; diff --git a/src/torrust_http_tracker/mod.rs b/src/http/mod.rs similarity index 100% rename from src/torrust_http_tracker/mod.rs rename to src/http/mod.rs diff --git a/src/torrust_http_tracker/request.rs b/src/http/request.rs similarity index 94% rename from src/torrust_http_tracker/request.rs rename to src/http/request.rs index 487e53a13..28cd4750e 100644 --- a/src/torrust_http_tracker/request.rs +++ b/src/http/request.rs @@ -3,7 +3,7 @@ use std::net::IpAddr; use serde::Deserialize; use crate::{InfoHash, PeerId}; -use crate::torrust_http_tracker::Bytes; +use crate::http::Bytes; #[derive(Deserialize)] pub struct AnnounceRequestQuery { diff --git a/src/torrust_http_tracker/response.rs b/src/http/response.rs similarity index 100% rename from src/torrust_http_tracker/response.rs rename to src/http/response.rs diff --git a/src/torrust_http_tracker/routes.rs b/src/http/routes.rs similarity index 90% rename from src/torrust_http_tracker/routes.rs rename to src/http/routes.rs index fb6bf5c16..775d9ce79 100644 --- a/src/torrust_http_tracker/routes.rs +++ b/src/http/routes.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use warp::{Filter, Rejection}; use crate::TorrentTracker; -use crate::torrust_http_tracker::{handle_announce, handle_scrape, send_error, with_announce_request, with_auth_key, with_scrape_request, with_tracker}; +use crate::http::{handle_announce, handle_scrape, send_error, with_announce_request, with_auth_key, with_scrape_request, with_tracker}; /// All routes pub fn routes(tracker: Arc) -> impl Filter + Clone { diff --git a/src/torrust_http_tracker/server.rs b/src/http/server.rs similarity index 97% rename from src/torrust_http_tracker/server.rs rename to src/http/server.rs index 336670030..31a8e4664 100644 --- a/src/torrust_http_tracker/server.rs +++ b/src/http/server.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; use std::sync::Arc; use crate::TorrentTracker; -use crate::torrust_http_tracker::routes; +use crate::http::routes; /// Server that listens on HTTP, needs a TorrentTracker #[derive(Clone)] diff --git a/src/lib.rs b/src/lib.rs index b6cebfc5e..4f8e9a241 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ -pub use torrust_http_tracker::server::*; -pub use torrust_udp_tracker::server::*; +pub use http::server::*; +pub use udp::server::*; pub use self::common::*; pub use self::config::*; @@ -14,8 +14,8 @@ pub mod utils; pub mod sqlite_database; pub mod key_manager; pub mod logging; -pub mod torrust_udp_tracker; -pub mod torrust_http_tracker; +pub mod udp; +pub mod http; pub mod database; pub mod mysql_database; pub mod torrent; diff --git a/src/main.rs b/src/main.rs index 5e16f2e0c..b2bcb31d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,7 @@ use log::info; use tokio::task::JoinHandle; use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; -use torrust_tracker::torrust_http_tracker::server::HttpServer; +use torrust_tracker::http::server::HttpServer; #[tokio::main] async fn main() { diff --git a/src/torrent.rs b/src/torrent.rs index e2984a490..4e44a995a 100644 --- a/src/torrent.rs +++ b/src/torrent.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; use crate::common::{AnnounceEventDef, NumberOfBytesDef}; -use crate::torrust_http_tracker::AnnounceRequest; +use crate::http::AnnounceRequest; #[derive(PartialEq, Eq, Debug, Clone, Serialize)] pub struct TorrentPeer { diff --git a/src/torrust_udp_tracker/errors.rs b/src/udp/errors.rs similarity index 100% rename from src/torrust_udp_tracker/errors.rs rename to src/udp/errors.rs diff --git a/src/torrust_udp_tracker/handlers.rs b/src/udp/handlers.rs similarity index 98% rename from src/torrust_udp_tracker/handlers.rs rename to src/udp/handlers.rs index ff6e8981b..23fac0405 100644 --- a/src/torrust_udp_tracker/handlers.rs +++ b/src/udp/handlers.rs @@ -5,8 +5,8 @@ use aquatic_udp_protocol::{AnnounceInterval, AnnounceRequest, AnnounceResponse, use crate::{InfoHash, MAX_SCRAPE_TORRENTS, TorrentTracker}; use crate::torrent::{TorrentError, TorrentPeer}; -use crate::torrust_udp_tracker::errors::ServerError; -use crate::torrust_udp_tracker::request::AnnounceRequestWrapper; +use crate::udp::errors::ServerError; +use crate::udp::request::AnnounceRequestWrapper; use crate::tracker_stats::TrackerStatsEvent; use crate::utils::get_connection_id; diff --git a/src/torrust_udp_tracker/mod.rs b/src/udp/mod.rs similarity index 100% rename from src/torrust_udp_tracker/mod.rs rename to src/udp/mod.rs diff --git a/src/torrust_udp_tracker/request.rs b/src/udp/request.rs similarity index 100% rename from src/torrust_udp_tracker/request.rs rename to src/udp/request.rs diff --git a/src/torrust_udp_tracker/server.rs b/src/udp/server.rs similarity index 97% rename from src/torrust_udp_tracker/server.rs rename to src/udp/server.rs index 8dc34d85d..03745192b 100644 --- a/src/torrust_udp_tracker/server.rs +++ b/src/udp/server.rs @@ -7,7 +7,7 @@ use log::{debug, info}; use tokio::net::UdpSocket; use crate::TorrentTracker; -use crate::torrust_udp_tracker::{handle_packet, MAX_PACKET_SIZE}; +use crate::udp::{handle_packet, MAX_PACKET_SIZE}; pub struct UdpServer { socket: Arc, From 5a6eaa41a319bff79e4ab2d291fb5b9d99dfef75 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 5 May 2022 23:58:10 +0200 Subject: [PATCH 36/53] refactor: moved databases to own module --- src/config.rs | 2 +- src/{ => databases}/database.rs | 4 ++-- src/databases/mod.rs | 3 +++ src/{mysql_database.rs => databases/mysql.rs} | 5 +++-- src/{sqlite_database.rs => databases/sqlite.rs} | 5 +++-- src/lib.rs | 7 +++---- src/tracker.rs | 7 ++++--- 7 files changed, 19 insertions(+), 14 deletions(-) rename src/{ => databases}/database.rs (96%) create mode 100644 src/databases/mod.rs rename src/{mysql_database.rs => databases/mysql.rs} (98%) rename src/{sqlite_database.rs => databases/sqlite.rs} (98%) diff --git a/src/config.rs b/src/config.rs index ce3f59760..c6901564f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,7 +9,7 @@ use config::{Config, ConfigError, File}; use serde::{Deserialize, Serialize, Serializer}; use toml; -use crate::database::DatabaseDrivers; +use crate::databases::database::DatabaseDrivers; pub use crate::tracker::TrackerMode; #[derive(Serialize, Deserialize, PartialEq)] diff --git a/src/database.rs b/src/databases/database.rs similarity index 96% rename from src/database.rs rename to src/databases/database.rs index a90161e91..b39a0ada1 100644 --- a/src/database.rs +++ b/src/databases/database.rs @@ -7,8 +7,8 @@ use serde::{Deserialize, Serialize}; use crate::InfoHash; use crate::key_manager::AuthKey; -use crate::mysql_database::MysqlDatabase; -use crate::sqlite_database::SqliteDatabase; +use crate::databases::mysql::MysqlDatabase; +use crate::databases::sqlite::SqliteDatabase; use crate::torrent::TorrentEntry; #[derive(Serialize, Deserialize, Debug)] diff --git a/src/databases/mod.rs b/src/databases/mod.rs new file mode 100644 index 000000000..119e34816 --- /dev/null +++ b/src/databases/mod.rs @@ -0,0 +1,3 @@ +pub mod mysql; +pub mod sqlite; +pub mod database; diff --git a/src/mysql_database.rs b/src/databases/mysql.rs similarity index 98% rename from src/mysql_database.rs rename to src/databases/mysql.rs index 15e2de633..df85402b3 100644 --- a/src/mysql_database.rs +++ b/src/databases/mysql.rs @@ -8,8 +8,9 @@ use r2d2_mysql::mysql::{Opts, OptsBuilder, params, TxOpts}; use r2d2_mysql::mysql::prelude::Queryable; use r2d2_mysql::MysqlConnectionManager; -use crate::{AUTH_KEY_LENGTH, database, InfoHash}; -use crate::database::{Database, Error}; +use crate::{AUTH_KEY_LENGTH, InfoHash}; +use crate::databases::database::{Database, Error}; +use crate::databases::database; use crate::key_manager::AuthKey; use crate::torrent::TorrentEntry; diff --git a/src/sqlite_database.rs b/src/databases/sqlite.rs similarity index 98% rename from src/sqlite_database.rs rename to src/databases/sqlite.rs index 82bb9d4fc..b51fd6c51 100644 --- a/src/sqlite_database.rs +++ b/src/databases/sqlite.rs @@ -7,8 +7,9 @@ use r2d2::Pool; use r2d2_sqlite::SqliteConnectionManager; use r2d2_sqlite::rusqlite::NO_PARAMS; -use crate::{AUTH_KEY_LENGTH, database, InfoHash}; -use crate::database::Database; +use crate::{AUTH_KEY_LENGTH, InfoHash}; +use crate::databases::database::Database; +use crate::databases::database; use crate::key_manager::AuthKey; use crate::torrent::TorrentEntry; diff --git a/src/lib.rs b/src/lib.rs index 4f8e9a241..e965fa88d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,13 +11,12 @@ pub mod tracker; pub mod http_api_server; pub mod common; pub mod utils; -pub mod sqlite_database; pub mod key_manager; pub mod logging; pub mod udp; pub mod http; -pub mod database; -pub mod mysql_database; pub mod torrent; pub mod tracker_stats; - +pub mod setup; +pub mod persistent_torrent_statistics; +pub mod databases; diff --git a/src/tracker.rs b/src/tracker.rs index c0c25bc41..b8c2b9931 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -8,10 +8,11 @@ use serde::{Deserialize, Serialize}; use serde; use tokio::sync::{RwLock, RwLockReadGuard}; -use crate::{Configuration, database, key_manager}; +use crate::{Configuration, key_manager}; use crate::common::InfoHash; -use crate::database::Database; +use crate::databases::database::Database; use tokio::sync::mpsc::error::SendError; +use crate::databases::database; use crate::key_manager::AuthKey; use crate::key_manager::Error::KeyInvalid; use crate::torrent::{TorrentEntry, TorrentError, TorrentPeer, TorrentStats}; @@ -124,7 +125,7 @@ impl TorrentTracker { } // Loading the torrents into memory - pub async fn load_torrents(&self) -> Result<(), database::Error> { + pub async fn load_persistent_torrents(&self) -> Result<(), database::Error> { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { From 4bf0c8b1b35b6df3faba79ed022dc468611022ae Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Fri, 6 May 2022 23:21:51 +0200 Subject: [PATCH 37/53] refactor: major refactor of all code --- src/api/mod.rs | 1 + src/{http_api_server.rs => api/server.rs} | 20 +- src/config.rs | 6 +- src/databases/database.rs | 6 +- src/databases/mysql.rs | 6 +- src/databases/sqlite.rs | 6 +- src/http/filters.rs | 5 +- src/http/handlers.rs | 20 +- src/http/routes.rs | 10 +- src/http/server.rs | 16 +- src/jobs/http_tracker.rs | 27 +++ src/jobs/log_statistics.rs | 32 +++ src/jobs/mod.rs | 6 + src/jobs/persistent_torrent_statistics.rs | 38 ++++ src/jobs/torrent_cleanup.rs | 36 ++++ src/jobs/tracker_api.rs | 15 ++ src/jobs/udp_tracker.rs | 22 ++ src/lib.rs | 14 +- src/main.rs | 194 ++---------------- src/{ => protocol}/common.rs | 0 src/protocol/mod.rs | 2 + src/{ => protocol}/utils.rs | 0 src/setup.rs | 54 +++++ src/{key_manager.rs => tracker/key.rs} | 18 +- src/tracker/mod.rs | 5 + src/tracker/peer.rs | 83 ++++++++ .../statistics.rs} | 38 ++-- src/{ => tracker}/torrent.rs | 83 +------- src/{ => tracker}/tracker.rs | 67 +++--- src/udp/handlers.rs | 22 +- src/udp/server.rs | 9 +- 31 files changed, 475 insertions(+), 386 deletions(-) create mode 100644 src/api/mod.rs rename src/{http_api_server.rs => api/server.rs} (95%) create mode 100644 src/jobs/http_tracker.rs create mode 100644 src/jobs/log_statistics.rs create mode 100644 src/jobs/mod.rs create mode 100644 src/jobs/persistent_torrent_statistics.rs create mode 100644 src/jobs/torrent_cleanup.rs create mode 100644 src/jobs/tracker_api.rs create mode 100644 src/jobs/udp_tracker.rs rename src/{ => protocol}/common.rs (100%) create mode 100644 src/protocol/mod.rs rename src/{ => protocol}/utils.rs (100%) create mode 100644 src/setup.rs rename src/{key_manager.rs => tracker/key.rs} (85%) create mode 100644 src/tracker/mod.rs create mode 100644 src/tracker/peer.rs rename src/{tracker_stats.rs => tracker/statistics.rs} (67%) rename src/{ => tracker}/torrent.rs (57%) rename src/{ => tracker}/tracker.rs (90%) diff --git a/src/api/mod.rs b/src/api/mod.rs new file mode 100644 index 000000000..74f47ad34 --- /dev/null +++ b/src/api/mod.rs @@ -0,0 +1 @@ +pub mod server; diff --git a/src/http_api_server.rs b/src/api/server.rs similarity index 95% rename from src/http_api_server.rs rename to src/api/server.rs index 89505cb09..77264162c 100644 --- a/src/http_api_server.rs +++ b/src/api/server.rs @@ -1,14 +1,14 @@ use std::cmp::min; use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; use std::sync::Arc; use serde::{Deserialize, Serialize}; -use warp::{Filter, filters, reply, reply::Reply, serve, Server}; +use warp::{Filter, filters, reply, serve}; -use crate::torrent::TorrentPeer; -use crate::tracker::TorrentTracker; - -use super::common::*; +use crate::protocol::common::*; +use crate::peer::TorrentPeer; +use crate::tracker::tracker::TorrentTracker; #[derive(Deserialize, Debug)] struct TorrentInfoQuery { @@ -84,7 +84,7 @@ fn authenticate(tokens: HashMap) -> impl Filter) -> Server + Clone + Send + Sync + 'static> { +pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp::Future { // GET /api/torrents?offset=:u32&limit=:u32 // View torrent list let api_torrents = tracker.clone(); @@ -309,5 +309,11 @@ pub fn build_server(tracker: Arc) -> Server, - pub cleanup_interval: Option, + pub cleanup_interval: u64, pub cleanup_peerless: bool, pub external_ip: Option, pub announce_interval: u32, @@ -147,7 +147,7 @@ impl Configuration { statistics: true, persistence: false, persistence_interval: Some(900), - cleanup_interval: Some(600), + cleanup_interval: 600, cleanup_peerless: true, external_ip: Some(String::from("0.0.0.0")), announce_interval: 120, diff --git a/src/databases/database.rs b/src/databases/database.rs index b39a0ada1..fd9f2a19d 100644 --- a/src/databases/database.rs +++ b/src/databases/database.rs @@ -6,10 +6,10 @@ use log::debug; use serde::{Deserialize, Serialize}; use crate::InfoHash; -use crate::key_manager::AuthKey; +use crate::tracker::key::AuthKey; use crate::databases::mysql::MysqlDatabase; use crate::databases::sqlite::SqliteDatabase; -use crate::torrent::TorrentEntry; +use crate::tracker::torrent::TorrentEntry; #[derive(Serialize, Deserialize, Debug)] pub enum DatabaseDrivers { @@ -40,7 +40,7 @@ pub fn connect_database(db_driver: &DatabaseDrivers, db_path: &str) -> Result Result<(), Error>; - async fn load_persistent_torrent_data(&self) -> Result, Error>; + async fn load_persistent_torrents(&self) -> Result, Error>; async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), Error>; diff --git a/src/databases/mysql.rs b/src/databases/mysql.rs index df85402b3..e7f57a7a4 100644 --- a/src/databases/mysql.rs +++ b/src/databases/mysql.rs @@ -11,8 +11,8 @@ use r2d2_mysql::MysqlConnectionManager; use crate::{AUTH_KEY_LENGTH, InfoHash}; use crate::databases::database::{Database, Error}; use crate::databases::database; -use crate::key_manager::AuthKey; -use crate::torrent::TorrentEntry; +use crate::tracker::key::AuthKey; +use crate::tracker::torrent::TorrentEntry; pub struct MysqlDatabase { pool: Pool, @@ -65,7 +65,7 @@ impl Database for MysqlDatabase { Ok(()) } - async fn load_persistent_torrent_data(&self) -> Result, database::Error> { + async fn load_persistent_torrents(&self) -> Result, database::Error> { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT HEX(info_hash), completed FROM torrents", |(info_hash_string, completed): (String, u32)| { diff --git a/src/databases/sqlite.rs b/src/databases/sqlite.rs index b51fd6c51..18a1d5a28 100644 --- a/src/databases/sqlite.rs +++ b/src/databases/sqlite.rs @@ -10,8 +10,8 @@ use r2d2_sqlite::rusqlite::NO_PARAMS; use crate::{AUTH_KEY_LENGTH, InfoHash}; use crate::databases::database::Database; use crate::databases::database; -use crate::key_manager::AuthKey; -use crate::torrent::TorrentEntry; +use crate::tracker::key::AuthKey; +use crate::tracker::torrent::TorrentEntry; pub struct SqliteDatabase { pool: Pool, @@ -59,7 +59,7 @@ impl Database for SqliteDatabase { .map(|_| ()) } - async fn load_persistent_torrent_data(&self) -> Result, database::Error> { + async fn load_persistent_torrents(&self) -> Result, database::Error> { let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; let mut stmt = conn.prepare("SELECT info_hash, completed FROM torrents")?; diff --git a/src/http/filters.rs b/src/http/filters.rs index 5e0c3e068..8f3ee04c0 100644 --- a/src/http/filters.rs +++ b/src/http/filters.rs @@ -6,9 +6,10 @@ use std::sync::Arc; use log::debug; use warp::{Filter, reject, Rejection}; -use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId, TorrentTracker}; -use crate::key_manager::AuthKey; +use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; +use crate::tracker::key::AuthKey; use crate::http::{AnnounceRequest, AnnounceRequestQuery, ScrapeRequest, ServerError, WebResult}; +use crate::tracker::tracker::TorrentTracker; /// Pass Arc along pub fn with_tracker(tracker: Arc) -> impl Filter, ), Error=Infallible> + Clone { diff --git a/src/http/handlers.rs b/src/http/handlers.rs index c81c93d9b..d7e4859d9 100644 --- a/src/http/handlers.rs +++ b/src/http/handlers.rs @@ -7,12 +7,14 @@ use log::debug; use warp::{reject, Rejection, Reply}; use warp::http::Response; -use crate::{InfoHash, TorrentTracker}; -use crate::key_manager::AuthKey; -use crate::torrent::{TorrentError, TorrentPeer, TorrentStats}; +use crate::{InfoHash}; +use crate::tracker::key::AuthKey; +use crate::tracker::torrent::{TorrentError, TorrentStats}; use crate::http::{AnnounceRequest, AnnounceResponse, ErrorResponse, Peer, ScrapeRequest, ScrapeResponse, ScrapeResponseEntry, ServerError, WebResult}; -use crate::tracker_stats::TrackerStatsEvent; -use crate::utils::url_encode_bytes; +use crate::peer::TorrentPeer; +use crate::tracker::statistics::TrackerStatisticsEvent; +use crate::protocol::utils::url_encode_bytes; +use crate::tracker::tracker::TorrentTracker; /// Authenticate InfoHash using optional AuthKey pub async fn authenticate(info_hash: &InfoHash, auth_key: &Option, tracker: Arc) -> Result<(), ServerError> { @@ -51,8 +53,8 @@ pub async fn handle_announce(announce_request: AnnounceRequest, auth_key: Option // send stats event match announce_request.peer_addr { - IpAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp4Announce).await; } - IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp6Announce).await; } + IpAddr::V4(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Tcp4Announce).await; } + IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Tcp6Announce).await; } } send_announce_response(&announce_request, torrent_stats, peers, announce_interval, tracker.config.announce_interval_min) @@ -84,8 +86,8 @@ pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option { tracker.send_stats_event(TrackerStatsEvent::Tcp4Scrape).await; } - IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Tcp6Scrape).await; } + IpAddr::V4(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Tcp4Scrape).await; } + IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Tcp6Scrape).await; } } send_scrape_response(files) diff --git a/src/http/routes.rs b/src/http/routes.rs index 775d9ce79..a0b197f44 100644 --- a/src/http/routes.rs +++ b/src/http/routes.rs @@ -3,8 +3,14 @@ use std::sync::Arc; use warp::{Filter, Rejection}; -use crate::TorrentTracker; -use crate::http::{handle_announce, handle_scrape, send_error, with_announce_request, with_auth_key, with_scrape_request, with_tracker}; +use crate::http::handle_announce; +use crate::http::handle_scrape; +use crate::http::send_error; +use crate::http::with_announce_request; +use crate::http::with_auth_key; +use crate::http::with_scrape_request; +use crate::http::with_tracker; +use crate::tracker::tracker::TorrentTracker; /// All routes pub fn routes(tracker: Arc) -> impl Filter + Clone { diff --git a/src/http/server.rs b/src/http/server.rs index 31a8e4664..5a5b5f735 100644 --- a/src/http/server.rs +++ b/src/http/server.rs @@ -1,8 +1,8 @@ use std::net::SocketAddr; use std::sync::Arc; -use crate::TorrentTracker; use crate::http::routes; +use crate::tracker::tracker::TorrentTracker; /// Server that listens on HTTP, needs a TorrentTracker #[derive(Clone)] @@ -18,18 +18,19 @@ impl HttpServer { } /// Start the HttpServer - pub async fn start(&self, socket_addr: SocketAddr) { + pub fn start(&self, socket_addr: SocketAddr) -> impl warp::Future { let (_addr, server) = warp::serve(routes(self.tracker.clone())) .bind_with_graceful_shutdown(socket_addr, async move { tokio::signal::ctrl_c() .await - .expect("failed to listen to shutdown signal"); + .expect("Failed to listen to shutdown signal."); }); - tokio::task::spawn(server); + + server } /// Start the HttpServer in TLS mode - pub async fn start_tls(&self, socket_addr: SocketAddr, ssl_cert_path: &str, ssl_key_path: &str) { + pub fn start_tls(&self, socket_addr: SocketAddr, ssl_cert_path: String, ssl_key_path: String) -> impl warp::Future { let (_addr, server) = warp::serve(routes(self.tracker.clone())) .tls() .cert_path(ssl_cert_path) @@ -37,8 +38,9 @@ impl HttpServer { .bind_with_graceful_shutdown(socket_addr, async move { tokio::signal::ctrl_c() .await - .expect("failed to listen to shutdown signal"); + .expect("Failed to listen to shutdown signal."); }); - tokio::task::spawn(server); + + server } } diff --git a/src/jobs/http_tracker.rs b/src/jobs/http_tracker.rs new file mode 100644 index 000000000..85f64200f --- /dev/null +++ b/src/jobs/http_tracker.rs @@ -0,0 +1,27 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use log::{info, warn}; +use tokio::task::JoinHandle; +use crate::{HttpServer, HttpTrackerConfig}; +use crate::tracker::tracker::TorrentTracker; + +pub fn start_job(config: &HttpTrackerConfig, tracker: Arc) -> JoinHandle<()> { + let bind_addr = config.bind_address.parse::().unwrap(); + let ssl_enabled = config.ssl_enabled; + let ssl_cert_path = config.ssl_cert_path.clone(); + let ssl_key_path = config.ssl_key_path.clone(); + + tokio::spawn(async move { + let http_tracker = HttpServer::new(tracker); + + if !ssl_enabled { + info!("Starting HTTP server on: {}", bind_addr); + http_tracker.start(bind_addr).await; + } else if ssl_enabled && ssl_cert_path.is_some() && ssl_key_path.is_some() { + info!("Starting HTTPS server on: {} (TLS)", bind_addr); + http_tracker.start_tls(bind_addr, ssl_cert_path.unwrap(), ssl_key_path.unwrap()).await; + } else { + warn!("Could not start HTTP tracker on: {}, missing SSL Cert or Key!", bind_addr); + } + }) +} diff --git a/src/jobs/log_statistics.rs b/src/jobs/log_statistics.rs new file mode 100644 index 000000000..f62399a47 --- /dev/null +++ b/src/jobs/log_statistics.rs @@ -0,0 +1,32 @@ +use std::sync::Arc; +use log::info; +use tokio::task::JoinHandle; +use crate::{Configuration}; +use crate::tracker::tracker::TorrentTracker; + +pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { + let weak_tracker = std::sync::Arc::downgrade(&tracker); + let interval = config.log_interval.unwrap_or(60); + + tokio::spawn(async move { + let interval = std::time::Duration::from_secs(interval); + let mut interval = tokio::time::interval(interval); + interval.tick().await; + + loop { + tokio::select! { + _ = tokio::signal::ctrl_c() => { + info!("Stopping statistics logging job.."); + break; + } + _ = interval.tick() => { + if let Some(tracker) = weak_tracker.upgrade() { + tracker.post_log().await; + } else { + break; + } + } + } + } + }) +} diff --git a/src/jobs/mod.rs b/src/jobs/mod.rs new file mode 100644 index 000000000..a71fcb210 --- /dev/null +++ b/src/jobs/mod.rs @@ -0,0 +1,6 @@ +pub mod persistent_torrent_statistics; +pub mod torrent_cleanup; +pub mod tracker_api; +pub mod log_statistics; +pub mod http_tracker; +pub mod udp_tracker; diff --git a/src/jobs/persistent_torrent_statistics.rs b/src/jobs/persistent_torrent_statistics.rs new file mode 100644 index 000000000..7ebc80bdb --- /dev/null +++ b/src/jobs/persistent_torrent_statistics.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; +use log::info; +use tokio::task::JoinHandle; +use crate::{Configuration}; +use crate::tracker::tracker::TorrentTracker; + +pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { + let weak_tracker = std::sync::Arc::downgrade(&tracker); + let interval = config.persistence_interval.unwrap_or(900); + + tokio::spawn(async move { + let interval = std::time::Duration::from_secs(interval); + let mut interval = tokio::time::interval(interval); + interval.tick().await; + + // periodically save torrents to database + loop { + tokio::select! { + _ = tokio::signal::ctrl_c() => { + // Save before shutting down + tracker.periodic_saving().await; + info!("Stopping periodic torrent saving job.."); + break; + } + _ = interval.tick() => { + if let Some(tracker) = weak_tracker.upgrade() { + info!("Saving torrents to database..."); + tracker.periodic_saving().await; + info!("Periodic saving done."); + } else { + // If tracker no longer exists, stop job + break; + } + } + } + } + }) +} diff --git a/src/jobs/torrent_cleanup.rs b/src/jobs/torrent_cleanup.rs new file mode 100644 index 000000000..05e639728 --- /dev/null +++ b/src/jobs/torrent_cleanup.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; +use chrono::Utc; +use log::info; +use tokio::task::JoinHandle; +use crate::{Configuration}; +use crate::tracker::tracker::TorrentTracker; + +pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { + let weak_tracker = std::sync::Arc::downgrade(&tracker); + let interval = config.cleanup_interval; + + tokio::spawn(async move { + let interval = std::time::Duration::from_secs(interval); + let mut interval = tokio::time::interval(interval); + interval.tick().await; + + loop { + tokio::select! { + _ = tokio::signal::ctrl_c() => { + info!("Stopping torrent cleanup job.."); + break; + } + _ = interval.tick() => { + if let Some(tracker) = weak_tracker.upgrade() { + let start_time = Utc::now().time(); + info!("Cleaning up torrents.."); + tracker.cleanup_torrents().await; + info!("Cleaned up torrents in: {}ms", (Utc::now().time() - start_time).num_milliseconds()) + } else { + break; + } + } + } + } + }) +} diff --git a/src/jobs/tracker_api.rs b/src/jobs/tracker_api.rs new file mode 100644 index 000000000..476a87a6a --- /dev/null +++ b/src/jobs/tracker_api.rs @@ -0,0 +1,15 @@ +use std::sync::Arc; +use log::info; +use tokio::task::JoinHandle; +use crate::{Configuration}; +use crate::api::server; +use crate::tracker::tracker::TorrentTracker; + +pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { + let bind_addr = config.http_api.bind_address.parse::().expect("Tracker API bind_address invalid."); + info!("Starting Torrust API server on: {}", bind_addr); + + tokio::spawn(async move { + server::start(bind_addr, tracker).await; + }) +} diff --git a/src/jobs/udp_tracker.rs b/src/jobs/udp_tracker.rs new file mode 100644 index 000000000..32ef76ef4 --- /dev/null +++ b/src/jobs/udp_tracker.rs @@ -0,0 +1,22 @@ +use std::sync::Arc; +use log::{error, info, warn}; +use tokio::task::JoinHandle; +use crate::{UdpServer, UdpTrackerConfig}; +use crate::tracker::tracker::TorrentTracker; + +pub fn start_job(config: &UdpTrackerConfig, tracker: Arc) -> JoinHandle<()> { + let bind_addr = config.bind_address.clone(); + + tokio::spawn(async move { + match UdpServer::new(tracker, &bind_addr).await { + Ok(udp_server) => { + info!("Starting UDP server on: {}", bind_addr); + udp_server.start().await; + } + Err(e) => { + warn!("Could not start UDP tracker on: {}", bind_addr); + error!("{}", e); + } + } + }) +} diff --git a/src/lib.rs b/src/lib.rs index e965fa88d..245f4686c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,22 +1,18 @@ pub use http::server::*; pub use udp::server::*; -pub use self::common::*; +pub use protocol::common::*; pub use self::config::*; -pub use self::http_api_server::*; +pub use api::server::*; pub use self::tracker::*; pub mod config; pub mod tracker; -pub mod http_api_server; -pub mod common; -pub mod utils; -pub mod key_manager; pub mod logging; pub mod udp; pub mod http; -pub mod torrent; -pub mod tracker_stats; pub mod setup; -pub mod persistent_torrent_statistics; pub mod databases; +pub mod jobs; +pub mod api; +pub mod protocol; diff --git a/src/main.rs b/src/main.rs index b2bcb31d1..794cda4bb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,13 @@ -use std::net::SocketAddr; use std::sync::Arc; - use log::info; -use tokio::task::JoinHandle; - -use torrust_tracker::{Configuration, http_api_server, HttpApiConfig, HttpTrackerConfig, logging, TorrentTracker, UdpServer, UdpTrackerConfig}; -use torrust_tracker::http::server::HttpServer; +use torrust_tracker::Configuration; +use torrust_tracker::logging; +use torrust_tracker::setup; +use torrust_tracker::tracker::tracker::TorrentTracker; #[tokio::main] async fn main() { - // torrust config + // Initialize Torrust config let config = match Configuration::load_from_file() { Ok(config) => Arc::new(config), Err(error) => { @@ -17,184 +15,28 @@ async fn main() { } }; - // the singleton torrent tracker that gets passed to the HTTP and UDP server - let tracker = Arc::new(TorrentTracker::new(config.clone()).unwrap_or_else(|e| { - panic!("{}", e) - })); - - // initialize logging - logging::setup_logging(&config); - - // load persistent torrents if enabled - if config.persistence { - info!("Loading persistent torrents into memory..."); - if tracker.load_torrents().await.is_err() { - panic!("Could not load persistent torrents.") - }; - info!("Persistent torrents loaded."); - - let _torrent_periodic_job = start_torrent_periodic_job(config.clone(), tracker.clone()); - } - - // start torrent cleanup job (periodically removes old peers) - let _torrent_cleanup_job = start_torrent_cleanup_job(config.clone(), tracker.clone()); - - // start HTTP API server - if config.http_api.enabled { - let _api_server = start_api_server(&config.http_api, tracker.clone()); - } - - // used to send graceful shutdown signal to udp listeners - let (tx, rx) = tokio::sync::watch::channel(false); - let mut udp_server_handles = Vec::new(); - - // start the udp blocks - for udp_tracker in &config.udp_trackers { - if !udp_tracker.enabled { continue; } - - if tracker.is_private() { - panic!("Could not start UDP tracker on: {} while in {:?}. UDP is not safe for private trackers!", udp_tracker.bind_address, config.mode); - } - - udp_server_handles.push( - start_udp_tracker_server(&udp_tracker, tracker.clone(), rx.clone()).await - ) - } - - // start the http blocks - for http_tracker in &config.http_trackers { - if !http_tracker.enabled { continue; } - - // SSL requires a cert and a key - if http_tracker.ssl_enabled && !http_tracker.verify_ssl_cert_and_key_set() { - panic!("Could not start HTTP tracker on: {}, missing SSL Cert or Key!", http_tracker.bind_address); + // Initialize Torrust tracker + let tracker = match TorrentTracker::new(config.clone()) { + Ok(tracker) => Arc::new(tracker), + Err(error) => { + panic!("{}", error) } + }; - let _ = start_http_tracker_server(&http_tracker, tracker.clone()); - } + // Initialize logging + logging::setup_logging(&config); - // start a thread to post statistics - let _ = start_statistics_job(config.clone(), tracker.clone()); + // Run jobs + let jobs = setup::setup(&config, tracker.clone()).await; // handle the signals here tokio::select! { _ = tokio::signal::ctrl_c() => { info!("Torrust shutting down.."); - // send kill signal - let _ = tx.send(true); - - // await for all udp servers to shutdown - futures::future::join_all(udp_server_handles).await; - - // Save torrents if enabled - if config.persistence { - info!("Saving torrents into SQL from memory..."); - let _ = tracker.periodic_saving().await; - info!("Torrents saved"); - } + // Await for all jobs to shutdown + futures::future::join_all(jobs).await; + info!("Torrust successfully shutdown."); } } } - -fn start_torrent_periodic_job(config: Arc, tracker: Arc) -> JoinHandle<()> { - let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.persistence_interval.unwrap_or(900); - - tokio::spawn(async move { - let interval = std::time::Duration::from_secs(interval); - let mut interval = tokio::time::interval(interval); - interval.tick().await; // first tick is immediate... - // periodically call tracker.cleanup_torrents() - loop { - interval.tick().await; - if let Some(tracker) = weak_tracker.upgrade() { - info!("Executing periodic saving..."); - tracker.periodic_saving().await; - info!("Periodic saving done."); - } else { - break; - } - } - }) -} - -fn start_torrent_cleanup_job(config: Arc, tracker: Arc) -> JoinHandle<()> { - let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.cleanup_interval.unwrap_or(600); - - tokio::spawn(async move { - let interval = std::time::Duration::from_secs(interval); - let mut interval = tokio::time::interval(interval); - interval.tick().await; // first tick is immediate... - // periodically call tracker.cleanup_torrents() - loop { - interval.tick().await; - if let Some(tracker) = weak_tracker.upgrade() { - tracker.cleanup_torrents().await; - } else { - break; - } - } - }) -} - -fn start_statistics_job(config: Arc, tracker: Arc) -> JoinHandle<()> { - let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.log_interval.unwrap_or(60); - - tokio::spawn(async move { - let interval = std::time::Duration::from_secs(interval); - let mut interval = tokio::time::interval(interval); - interval.tick().await; // first tick is immediate... - // periodically call tracker.cleanup_torrents() - loop { - interval.tick().await; - if let Some(tracker) = weak_tracker.upgrade() { - tracker.post_log().await; - } else { - break; - } - } - }) -} - -fn start_api_server(config: &HttpApiConfig, tracker: Arc) -> JoinHandle<()> { - info!("Starting HTTP API server on: {}", config.bind_address); - let bind_addr = config.bind_address.parse::().unwrap(); - - tokio::spawn(async move { - let server = http_api_server::build_server(tracker); - let _ = server.bind(bind_addr).await; - }) -} - -fn start_http_tracker_server(config: &HttpTrackerConfig, tracker: Arc) -> JoinHandle<()> { - let http_tracker = HttpServer::new(tracker); - let bind_addr = config.bind_address.parse::().unwrap(); - let ssl_enabled = config.ssl_enabled; - let ssl_cert_path = config.ssl_cert_path.clone(); - let ssl_key_path = config.ssl_key_path.clone(); - - tokio::spawn(async move { - // run with tls if ssl_enabled and cert and key path are set - if ssl_enabled { - info!("Starting HTTPS server on: {} (TLS)", bind_addr); - http_tracker.start_tls(bind_addr, ssl_cert_path.as_ref().unwrap(), ssl_key_path.as_ref().unwrap()).await; - } else { - info!("Starting HTTP server on: {}", bind_addr); - http_tracker.start(bind_addr).await; - } - }) -} - -async fn start_udp_tracker_server(config: &UdpTrackerConfig, tracker: Arc, rx: tokio::sync::watch::Receiver) -> JoinHandle<()> { - let udp_server = UdpServer::new(tracker, &config.bind_address).await.unwrap_or_else(|e| { - panic!("Could not start UDP server: {}", e); - }); - - info!("Starting UDP server on: {}", config.bind_address); - tokio::spawn(async move { - udp_server.start(rx).await; - }) -} diff --git a/src/common.rs b/src/protocol/common.rs similarity index 100% rename from src/common.rs rename to src/protocol/common.rs diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs new file mode 100644 index 000000000..99cfd91e4 --- /dev/null +++ b/src/protocol/mod.rs @@ -0,0 +1,2 @@ +pub mod common; +pub mod utils; diff --git a/src/utils.rs b/src/protocol/utils.rs similarity index 100% rename from src/utils.rs rename to src/protocol/utils.rs diff --git a/src/setup.rs b/src/setup.rs new file mode 100644 index 000000000..b8d49614d --- /dev/null +++ b/src/setup.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; +use log::{info, warn}; +use tokio::task::JoinHandle; +use crate::{Configuration}; +use crate::jobs::{http_tracker, log_statistics, persistent_torrent_statistics, torrent_cleanup, tracker_api, udp_tracker}; +use crate::tracker::tracker::TorrentTracker; + +pub async fn setup(config: &Configuration, tracker: Arc) -> Vec>{ + let mut jobs: Vec> = Vec::new(); + + // Load persistent torrents + if config.persistence { + info!("Loading persistent torrents into memory.."); + tracker.load_persistent_torrents().await.expect("Could not load persistent torrents."); + info!("Persistent torrents loaded."); + jobs.push(persistent_torrent_statistics::start_job(&config, tracker.clone())); + } + + // Start the UDP blocks + for udp_tracker_config in &config.udp_trackers { + if !udp_tracker_config.enabled { continue; } + + if tracker.is_private() { + warn!("Could not start UDP tracker on: {} while in {:?}. UDP is not safe for private trackers!", udp_tracker_config.bind_address, config.mode); + } else { + jobs.push(udp_tracker::start_job(&udp_tracker_config, tracker.clone())) + } + } + + // Start the HTTP blocks + for http_tracker_config in &config.http_trackers { + if !http_tracker_config.enabled { continue; } + jobs.push(http_tracker::start_job(&http_tracker_config, tracker.clone())); + } + + // Start HTTP API server + if config.http_api.enabled { + jobs.push(tracker_api::start_job(&config, tracker.clone())); + } + + // Remove torrents without peers, every interval + if config.cleanup_interval > 0 { + jobs.push(torrent_cleanup::start_job(&config, tracker.clone())); + } + + // Log detailed torrent stats + if let Some(log_interval) = config.log_interval { + if log_interval > 0 { + jobs.push(log_statistics::start_job(&config, tracker.clone())); + } + } + + jobs +} diff --git a/src/key_manager.rs b/src/tracker/key.rs similarity index 85% rename from src/key_manager.rs rename to src/tracker/key.rs index 507402358..2e2ca81f7 100644 --- a/src/key_manager.rs +++ b/src/tracker/key.rs @@ -4,9 +4,9 @@ use rand::{Rng, thread_rng}; use rand::distributions::Alphanumeric; use serde::Serialize; -use crate::utils::current_time; +use crate::protocol::utils::current_time; -use super::common::AUTH_KEY_LENGTH; +use crate::AUTH_KEY_LENGTH; pub fn generate_auth_key(seconds_valid: u64) -> AuthKey { let key: String = thread_rng() @@ -81,11 +81,11 @@ impl From for Error { #[cfg(test)] mod tests { - use crate::key_manager; + use crate::tracker::key; #[test] fn auth_key_from_buffer() { - let auth_key = key_manager::AuthKey::from_buffer( + let auth_key = key::AuthKey::from_buffer( [ 89, 90, 83, 108, 52, 108, 77, 90, @@ -104,7 +104,7 @@ mod tests { #[test] fn auth_key_from_string() { let key_string = "YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ"; - let auth_key = key_manager::AuthKey::from_string(key_string); + let auth_key = key::AuthKey::from_string(key_string); assert!(auth_key.is_some()); assert_eq!(auth_key.unwrap().key, key_string); @@ -112,16 +112,16 @@ mod tests { #[test] fn generate_valid_auth_key() { - let auth_key = key_manager::generate_auth_key(9999); + let auth_key = key::generate_auth_key(9999); - assert!(key_manager::verify_auth_key(&auth_key).is_ok()); + assert!(key::verify_auth_key(&auth_key).is_ok()); } #[test] fn generate_expired_auth_key() { - let mut auth_key = key_manager::generate_auth_key(0); + let mut auth_key = key::generate_auth_key(0); auth_key.valid_until = Some(0); - assert!(key_manager::verify_auth_key(&auth_key).is_err()); + assert!(key::verify_auth_key(&auth_key).is_err()); } } diff --git a/src/tracker/mod.rs b/src/tracker/mod.rs new file mode 100644 index 000000000..6115bc23e --- /dev/null +++ b/src/tracker/mod.rs @@ -0,0 +1,5 @@ +pub mod tracker; +pub mod statistics; +pub mod peer; +pub mod torrent; +pub mod key; diff --git a/src/tracker/peer.rs b/src/tracker/peer.rs new file mode 100644 index 000000000..2a54e2fd2 --- /dev/null +++ b/src/tracker/peer.rs @@ -0,0 +1,83 @@ +use std::net::{IpAddr, SocketAddr}; +use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; +use serde; +use serde::{Serialize}; +use crate::protocol::common::{NumberOfBytesDef, AnnounceEventDef}; +use crate::http::AnnounceRequest; +use crate::PeerId; + +#[derive(PartialEq, Eq, Debug, Clone, Serialize)] +pub struct TorrentPeer { + pub peer_id: PeerId, + pub peer_addr: SocketAddr, + #[serde(serialize_with = "ser_instant")] + pub updated: std::time::Instant, + #[serde(with = "NumberOfBytesDef")] + pub uploaded: NumberOfBytes, + #[serde(with = "NumberOfBytesDef")] + pub downloaded: NumberOfBytes, + #[serde(with = "NumberOfBytesDef")] + pub left: NumberOfBytes, + #[serde(with = "AnnounceEventDef")] + pub event: AnnounceEvent, +} + +impl TorrentPeer { + pub fn from_udp_announce_request(announce_request: &aquatic_udp_protocol::AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { + let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port.0); + + TorrentPeer { + peer_id: PeerId(announce_request.peer_id.0), + peer_addr, + updated: std::time::Instant::now(), + uploaded: announce_request.bytes_uploaded, + downloaded: announce_request.bytes_downloaded, + left: announce_request.bytes_left, + event: announce_request.event, + } + } + + pub fn from_http_announce_request(announce_request: &AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { + let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port); + + let event: AnnounceEvent = if let Some(event) = &announce_request.event { + match event.as_ref() { + "started" => AnnounceEvent::Started, + "stopped" => AnnounceEvent::Stopped, + "completed" => AnnounceEvent::Completed, + _ => AnnounceEvent::None + } + } else { + AnnounceEvent::None + }; + + TorrentPeer { + peer_id: announce_request.peer_id.clone(), + peer_addr, + updated: std::time::Instant::now(), + uploaded: NumberOfBytes(announce_request.uploaded as i64), + downloaded: NumberOfBytes(announce_request.downloaded as i64), + left: NumberOfBytes(announce_request.left as i64), + event, + } + } + + // potentially substitute localhost ip with external ip + pub fn peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip: IpAddr, host_opt_ip: Option, port: u16) -> SocketAddr { + if remote_ip.is_loopback() && host_opt_ip.is_some() { + SocketAddr::new(host_opt_ip.unwrap(), port) + } else { + SocketAddr::new(remote_ip, port) + } + } + + pub(crate) fn is_seeder(&self) -> bool { self.left.0 <= 0 && self.event != AnnounceEvent::Stopped } + + pub(crate) fn is_completed(&self) -> bool { + self.event == AnnounceEvent::Completed + } +} + +fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { + ser.serialize_u64(inst.elapsed().as_millis() as u64) +} diff --git a/src/tracker_stats.rs b/src/tracker/statistics.rs similarity index 67% rename from src/tracker_stats.rs rename to src/tracker/statistics.rs index 0bcd781ba..5822d2d4e 100644 --- a/src/tracker_stats.rs +++ b/src/tracker/statistics.rs @@ -7,7 +7,7 @@ use tokio::sync::mpsc::error::SendError; const CHANNEL_BUFFER_SIZE: usize = 65_535; #[derive(Debug)] -pub enum TrackerStatsEvent { +pub enum TrackerStatisticsEvent { Tcp4Announce, Tcp4Scrape, Tcp6Announce, @@ -21,7 +21,7 @@ pub enum TrackerStatsEvent { } #[derive(Debug)] -pub struct TrackerStats { +pub struct TrackerStatistics { pub tcp4_connections_handled: u64, pub tcp4_announces_handled: u64, pub tcp4_scrapes_handled: u64, @@ -36,7 +36,7 @@ pub struct TrackerStats { pub udp6_scrapes_handled: u64, } -impl TrackerStats { +impl TrackerStatistics { pub fn new() -> Self { Self { tcp4_connections_handled: 0, @@ -56,23 +56,23 @@ impl TrackerStats { } pub struct StatsTracker { - channel_sender: Option>, - pub stats: Arc>, + channel_sender: Option>, + pub stats: Arc>, } impl StatsTracker { pub fn new() -> Self { Self { channel_sender: None, - stats: Arc::new(RwLock::new(TrackerStats::new())), + stats: Arc::new(RwLock::new(TrackerStatistics::new())), } } - pub async fn get_stats(&self) -> RwLockReadGuard<'_, TrackerStats> { + pub async fn get_stats(&self) -> RwLockReadGuard<'_, TrackerStatistics> { self.stats.read().await } - pub async fn send_event(&self, event: TrackerStatsEvent) -> Option>> { + pub async fn send_event(&self, event: TrackerStatisticsEvent) -> Option>> { if let Some(tx) = &self.channel_sender { Some(tx.send(event).await) } else { @@ -81,7 +81,7 @@ impl StatsTracker { } pub fn run_worker(&mut self) { - let (tx, mut rx) = mpsc::channel::(CHANNEL_BUFFER_SIZE); + let (tx, mut rx) = mpsc::channel::(CHANNEL_BUFFER_SIZE); // set send channel on stats_tracker self.channel_sender = Some(tx); @@ -93,28 +93,28 @@ impl StatsTracker { let mut stats_lock = stats.write().await; match event { - TrackerStatsEvent::Tcp4Announce => { + TrackerStatisticsEvent::Tcp4Announce => { stats_lock.tcp4_announces_handled += 1; stats_lock.tcp4_connections_handled += 1; } - TrackerStatsEvent::Tcp4Scrape => { + TrackerStatisticsEvent::Tcp4Scrape => { stats_lock.tcp4_scrapes_handled += 1; stats_lock.tcp4_connections_handled += 1; } - TrackerStatsEvent::Tcp6Announce => { + TrackerStatisticsEvent::Tcp6Announce => { stats_lock.tcp6_announces_handled += 1; stats_lock.tcp6_connections_handled += 1; } - TrackerStatsEvent::Tcp6Scrape => { + TrackerStatisticsEvent::Tcp6Scrape => { stats_lock.tcp6_scrapes_handled += 1; stats_lock.tcp6_connections_handled += 1; } - TrackerStatsEvent::Udp4Connect => { stats_lock.udp4_connections_handled += 1; } - TrackerStatsEvent::Udp4Announce => { stats_lock.udp4_announces_handled += 1; } - TrackerStatsEvent::Udp4Scrape => { stats_lock.udp4_scrapes_handled += 1; } - TrackerStatsEvent::Udp6Connect => { stats_lock.udp6_connections_handled += 1; } - TrackerStatsEvent::Udp6Announce => { stats_lock.udp6_announces_handled += 1; } - TrackerStatsEvent::Udp6Scrape => { stats_lock.udp6_scrapes_handled += 1; } + TrackerStatisticsEvent::Udp4Connect => { stats_lock.udp4_connections_handled += 1; } + TrackerStatisticsEvent::Udp4Announce => { stats_lock.udp4_announces_handled += 1; } + TrackerStatisticsEvent::Udp4Scrape => { stats_lock.udp4_scrapes_handled += 1; } + TrackerStatisticsEvent::Udp6Connect => { stats_lock.udp6_connections_handled += 1; } + TrackerStatisticsEvent::Udp6Announce => { stats_lock.udp6_announces_handled += 1; } + TrackerStatisticsEvent::Udp6Scrape => { stats_lock.udp6_scrapes_handled += 1; } } drop(stats_lock); diff --git a/src/torrent.rs b/src/tracker/torrent.rs similarity index 57% rename from src/torrent.rs rename to src/tracker/torrent.rs index 4e44a995a..c5c721a4a 100644 --- a/src/torrent.rs +++ b/src/tracker/torrent.rs @@ -1,88 +1,11 @@ use std::borrow::Cow; -use std::net::{IpAddr, SocketAddr}; +use std::net::{IpAddr}; -use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; +use aquatic_udp_protocol::{AnnounceEvent}; use serde::{Deserialize, Serialize}; use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; -use crate::common::{AnnounceEventDef, NumberOfBytesDef}; -use crate::http::AnnounceRequest; - -#[derive(PartialEq, Eq, Debug, Clone, Serialize)] -pub struct TorrentPeer { - pub peer_id: PeerId, - pub peer_addr: SocketAddr, - #[serde(serialize_with = "ser_instant")] - pub updated: std::time::Instant, - #[serde(with = "NumberOfBytesDef")] - pub uploaded: NumberOfBytes, - #[serde(with = "NumberOfBytesDef")] - pub downloaded: NumberOfBytes, - #[serde(with = "NumberOfBytesDef")] - pub left: NumberOfBytes, - #[serde(with = "AnnounceEventDef")] - pub event: AnnounceEvent, -} - -impl TorrentPeer { - pub fn from_udp_announce_request(announce_request: &aquatic_udp_protocol::AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { - let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port.0); - - TorrentPeer { - peer_id: PeerId(announce_request.peer_id.0), - peer_addr, - updated: std::time::Instant::now(), - uploaded: announce_request.bytes_uploaded, - downloaded: announce_request.bytes_downloaded, - left: announce_request.bytes_left, - event: announce_request.event, - } - } - - pub fn from_http_announce_request(announce_request: &AnnounceRequest, remote_ip: IpAddr, host_opt_ip: Option) -> Self { - let peer_addr = TorrentPeer::peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip, host_opt_ip, announce_request.port); - - let event: AnnounceEvent = if let Some(event) = &announce_request.event { - match event.as_ref() { - "started" => AnnounceEvent::Started, - "stopped" => AnnounceEvent::Stopped, - "completed" => AnnounceEvent::Completed, - _ => AnnounceEvent::None - } - } else { - AnnounceEvent::None - }; - - TorrentPeer { - peer_id: announce_request.peer_id.clone(), - peer_addr, - updated: std::time::Instant::now(), - uploaded: NumberOfBytes(announce_request.uploaded as i64), - downloaded: NumberOfBytes(announce_request.downloaded as i64), - left: NumberOfBytes(announce_request.left as i64), - event, - } - } - - // potentially substitute localhost ip with external ip - pub fn peer_addr_from_ip_and_port_and_opt_host_ip(remote_ip: IpAddr, host_opt_ip: Option, port: u16) -> SocketAddr { - if remote_ip.is_loopback() && host_opt_ip.is_some() { - SocketAddr::new(host_opt_ip.unwrap(), port) - } else { - SocketAddr::new(remote_ip, port) - } - } - - pub(crate) fn is_seeder(&self) -> bool { self.left.0 <= 0 && self.event != AnnounceEvent::Stopped } - - fn is_completed(&self) -> bool { - self.event == AnnounceEvent::Completed - } -} - -fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { - ser.serialize_u64(inst.elapsed().as_millis() as u64) -} +use crate::peer::TorrentPeer; #[derive(Serialize, Deserialize, Clone)] pub struct TorrentEntry { diff --git a/src/tracker.rs b/src/tracker/tracker.rs similarity index 90% rename from src/tracker.rs rename to src/tracker/tracker.rs index b8c2b9931..4f0d571c6 100644 --- a/src/tracker.rs +++ b/src/tracker/tracker.rs @@ -3,20 +3,22 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use log::{debug, info}; +use log::info; use serde::{Deserialize, Serialize}; use serde; use tokio::sync::{RwLock, RwLockReadGuard}; -use crate::{Configuration, key_manager}; -use crate::common::InfoHash; +use crate::Configuration; +use crate::protocol::common::InfoHash; use crate::databases::database::Database; use tokio::sync::mpsc::error::SendError; use crate::databases::database; -use crate::key_manager::AuthKey; -use crate::key_manager::Error::KeyInvalid; -use crate::torrent::{TorrentEntry, TorrentError, TorrentPeer, TorrentStats}; -use crate::tracker_stats::{StatsTracker, TrackerStats, TrackerStatsEvent}; +use crate::peer::TorrentPeer; +use crate::tracker::key::AuthKey; +use crate::tracker::key::Error::KeyInvalid; +use crate::statistics::{StatsTracker, TrackerStatistics, TrackerStatisticsEvent}; +use crate::tracker::key; +use crate::tracker::torrent::{TorrentEntry, TorrentError, TorrentStats}; #[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Debug)] pub enum TrackerMode { @@ -79,7 +81,7 @@ impl TorrentTracker { } pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { - let auth_key = key_manager::generate_auth_key(seconds_valid); + let auth_key = key::generate_auth_key(seconds_valid); // add key to database if let Err(error) = self.database.add_key_to_keys(&auth_key).await { return Err(error); } @@ -91,9 +93,9 @@ impl TorrentTracker { self.database.remove_key_from_keys(key).await } - pub async fn verify_auth_key(&self, auth_key: &AuthKey) -> Result<(), key_manager::Error> { + pub async fn verify_auth_key(&self, auth_key: &AuthKey) -> Result<(), key::Error> { let db_key = self.database.get_key_from_keys(&auth_key.key).await.map_err(|_| KeyInvalid)?; - key_manager::verify_auth_key(&db_key) + key::verify_auth_key(&db_key) } pub async fn authenticate_request(&self, info_hash: &InfoHash, key: &Option) -> Result<(), TorrentError> { @@ -124,13 +126,22 @@ impl TorrentTracker { Ok(()) } - // Loading the torrents into memory + // Loading the torrents from database into memory pub async fn load_persistent_torrents(&self) -> Result<(), database::Error> { - let torrents = self.database.load_persistent_torrent_data().await?; + let persistent_torrents = self.database.load_persistent_torrents().await?; + let mut torrents = self.torrents.write().await; + + for (info_hash, completed) in persistent_torrents { + // Skip if torrent entry already exists + if torrents.contains_key(&info_hash) { continue; } + + let torrent_entry = TorrentEntry { + peers: Default::default(), + completed, + seeders: Default::default(), + }; - for torrent in torrents { - debug!("{:#?}", torrent); - let _ = self.add_torrent(torrent.0, 0, torrent.1, 0).await; + torrents.insert(info_hash.clone(), torrent_entry); } Ok(()) @@ -206,34 +217,15 @@ impl TorrentTracker { } } - pub async fn add_torrent(&self, info_hash: InfoHash, seeders: u32, completed: u32, leechers: u32) -> TorrentStats { - let mut torrents = self.torrents.write().await; - - if !torrents.contains_key(&info_hash) { - let torrent_entry = TorrentEntry { - peers: Default::default(), - completed, - seeders, - }; - torrents.insert(info_hash.clone(), torrent_entry); - } - - TorrentStats { - seeders, - completed, - leechers, - } - } - pub async fn get_torrents(&self) -> RwLockReadGuard<'_, BTreeMap> { self.torrents.read().await } - pub async fn get_stats(&self) -> RwLockReadGuard<'_, TrackerStats> { + pub async fn get_stats(&self) -> RwLockReadGuard<'_, TrackerStatistics> { self.stats_tracker.get_stats().await } - pub async fn send_stats_event(&self, event: TrackerStatsEvent) -> Option>> { + pub async fn send_stats_event(&self, event: TrackerStatisticsEvent) -> Option>> { self.stats_tracker.send_event(event).await } @@ -252,8 +244,6 @@ impl TorrentTracker { // remove torrents without peers if enabled, and defragment memory pub async fn cleanup_torrents(&self) { - info!("Cleaning torrents..."); - let lock = self.torrents.write().await; // First we create a mapping of all the torrent hashes in a vector, and we use this to iterate through the btreemap. @@ -298,7 +288,6 @@ impl TorrentTracker { } drop(lock); } - info!("Torrents cleaned up."); } pub async fn periodic_saving(&self) { diff --git a/src/udp/handlers.rs b/src/udp/handlers.rs index 23fac0405..4ea767c0b 100644 --- a/src/udp/handlers.rs +++ b/src/udp/handlers.rs @@ -3,12 +3,14 @@ use std::sync::Arc; use aquatic_udp_protocol::{AnnounceInterval, AnnounceRequest, AnnounceResponse, ConnectRequest, ConnectResponse, ErrorResponse, NumberOfDownloads, NumberOfPeers, Port, Request, Response, ResponsePeer, ScrapeRequest, ScrapeResponse, TorrentScrapeStatistics, TransactionId}; -use crate::{InfoHash, MAX_SCRAPE_TORRENTS, TorrentTracker}; -use crate::torrent::{TorrentError, TorrentPeer}; +use crate::{InfoHash, MAX_SCRAPE_TORRENTS}; +use crate::peer::TorrentPeer; +use crate::tracker::torrent::{TorrentError}; use crate::udp::errors::ServerError; use crate::udp::request::AnnounceRequestWrapper; -use crate::tracker_stats::TrackerStatsEvent; -use crate::utils::get_connection_id; +use crate::tracker::statistics::TrackerStatisticsEvent; +use crate::tracker::tracker::TorrentTracker; +use crate::protocol::utils::get_connection_id; pub async fn authenticate(info_hash: &InfoHash, tracker: Arc) -> Result<(), ServerError> { match tracker.authenticate_request(info_hash, &None).await { @@ -77,8 +79,8 @@ pub async fn handle_connect(remote_addr: SocketAddr, request: &ConnectRequest, t // send stats event match remote_addr { - SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp4Connect).await; } - SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp6Connect).await; } + SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp4Connect).await; } + SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp6Connect).await; } } Ok(response) @@ -136,8 +138,8 @@ pub async fn handle_announce(remote_addr: SocketAddr, announce_request: &Announc // send stats event match remote_addr { - SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp4Announce).await; } - SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp6Announce).await; } + SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp4Announce).await; } + SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp6Announce).await; } } Ok(announce_response) @@ -178,8 +180,8 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra // send stats event match remote_addr { - SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp4Scrape).await; } - SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatsEvent::Udp6Scrape).await; } + SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp4Scrape).await; } + SocketAddr::V6(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp6Scrape).await; } } Ok(Response::from(ScrapeResponse { diff --git a/src/udp/server.rs b/src/udp/server.rs index 03745192b..bcacc2642 100644 --- a/src/udp/server.rs +++ b/src/udp/server.rs @@ -6,7 +6,7 @@ use aquatic_udp_protocol::Response; use log::{debug, info}; use tokio::net::UdpSocket; -use crate::TorrentTracker; +use crate::tracker::tracker::TorrentTracker; use crate::udp::{handle_packet, MAX_PACKET_SIZE}; pub struct UdpServer { @@ -24,16 +24,15 @@ impl UdpServer { }) } - pub async fn start(&self, rx: tokio::sync::watch::Receiver) { + pub async fn start(&self) { loop { - let mut rx = rx.clone(); let mut data = [0; MAX_PACKET_SIZE]; let socket = self.socket.clone(); let tracker = self.tracker.clone(); tokio::select! { - _ = rx.changed() => { - info!("Stopping UDP server: {}...", socket.local_addr().unwrap()); + _ = tokio::signal::ctrl_c() => { + info!("Stopping UDP server: {}..", socket.local_addr().unwrap()); break; } Ok((valid_bytes, remote_addr)) = socket.recv_from(&mut data) => { From 047c2db9100ae0d52af57d786c15aa237f1d1e03 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 01:45:00 +0200 Subject: [PATCH 38/53] refactor: refactored even more code --- src/api/server.rs | 4 +- src/config.rs | 4 +- src/http/response.rs | 2 + src/protocol/utils.rs | 4 ++ src/tracker/mod.rs | 1 + src/tracker/mode.rs | 21 ++++++++ src/tracker/peer.rs | 13 ++--- src/tracker/torrent.rs | 119 ++++++++++------------------------------- src/tracker/tracker.rs | 59 ++++++-------------- 9 files changed, 80 insertions(+), 147 deletions(-) create mode 100644 src/tracker/mode.rs diff --git a/src/api/server.rs b/src/api/server.rs index 77264162c..d33f17dc0 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -23,7 +23,7 @@ struct Torrent<'a> { completed: u32, leechers: u32, #[serde(skip_serializing_if = "Option::is_none")] - peers: Option>, + peers: Option>, } #[derive(Serialize)] @@ -198,7 +198,7 @@ pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp let torrent_entry_option = db.get(&info_hash); if torrent_entry_option.is_none() { - return Err(warp::reject::custom(ActionStatus::Err { reason: "torrent does not exist".into() })); + return Result::<_, warp::reject::Rejection>::Ok(reply::json(&"torrent not known")) } let torrent_entry = torrent_entry_option.unwrap(); diff --git a/src/config.rs b/src/config.rs index 10268ebbb..c24d1c45d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize, Serializer}; use toml; use crate::databases::database::DatabaseDrivers; -use crate::tracker::tracker::TrackerMode; +use crate::mode::TrackerMode; #[derive(Serialize, Deserialize, PartialEq)] pub enum TrackerServer { @@ -141,7 +141,7 @@ impl Configuration { let mut configuration = Configuration { log_level: Option::from(String::from("info")), log_interval: Some(60), - mode: TrackerMode::PublicMode, + mode: TrackerMode::Public, db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), statistics: true, diff --git a/src/http/response.rs b/src/http/response.rs index f57129cde..3118f7df1 100644 --- a/src/http/response.rs +++ b/src/http/response.rs @@ -3,6 +3,7 @@ use std::error::Error; use std::io::Write; use std::net::IpAddr; +use serde; use serde::Serialize; #[derive(Serialize)] @@ -15,6 +16,7 @@ pub struct Peer { #[derive(Serialize)] pub struct AnnounceResponse { pub interval: u32, + #[serde(rename = "min interval")] pub interval_min: u32, //pub tracker_id: String, pub complete: u32, diff --git a/src/protocol/utils.rs b/src/protocol/utils.rs index fb2a94513..5fc694c8e 100644 --- a/src/protocol/utils.rs +++ b/src/protocol/utils.rs @@ -57,3 +57,7 @@ pub async fn convert_bytes_to_int(array: &Vec) -> u64 { let mut rdr = Cursor::new(array_fixed); return rdr.read_u64::().unwrap(); } + +pub fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { + ser.serialize_u64(inst.elapsed().as_millis() as u64) +} diff --git a/src/tracker/mod.rs b/src/tracker/mod.rs index 6115bc23e..791e2e7d2 100644 --- a/src/tracker/mod.rs +++ b/src/tracker/mod.rs @@ -3,3 +3,4 @@ pub mod statistics; pub mod peer; pub mod torrent; pub mod key; +pub mod mode; diff --git a/src/tracker/mode.rs b/src/tracker/mode.rs new file mode 100644 index 000000000..edcb27f1c --- /dev/null +++ b/src/tracker/mode.rs @@ -0,0 +1,21 @@ +use serde; +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Debug)] +pub enum TrackerMode { + // Will track every new info hash and serve every peer. + #[serde(rename = "public")] + Public, + + // Will only track whitelisted info hashes. + #[serde(rename = "listed")] + Listed, + + // Will only serve authenticated peers + #[serde(rename = "private")] + Private, + + // Will only track whitelisted info hashes and serve authenticated peers + #[serde(rename = "private_listed")] + PrivateListed, +} diff --git a/src/tracker/peer.rs b/src/tracker/peer.rs index 2a54e2fd2..ce4e52022 100644 --- a/src/tracker/peer.rs +++ b/src/tracker/peer.rs @@ -1,8 +1,11 @@ use std::net::{IpAddr, SocketAddr}; + use aquatic_udp_protocol::{AnnounceEvent, NumberOfBytes}; use serde; use serde::{Serialize}; + use crate::protocol::common::{NumberOfBytesDef, AnnounceEventDef}; +use crate::protocol::utils::ser_instant; use crate::http::AnnounceRequest; use crate::PeerId; @@ -71,13 +74,5 @@ impl TorrentPeer { } } - pub(crate) fn is_seeder(&self) -> bool { self.left.0 <= 0 && self.event != AnnounceEvent::Stopped } - - pub(crate) fn is_completed(&self) -> bool { - self.event == AnnounceEvent::Completed - } -} - -fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { - ser.serialize_u64(inst.elapsed().as_millis() as u64) + pub fn is_seeder(&self) -> bool { self.left.0 <= 0 && self.event != AnnounceEvent::Stopped } } diff --git a/src/tracker/torrent.rs b/src/tracker/torrent.rs index c5c721a4a..a01b5ce55 100644 --- a/src/tracker/torrent.rs +++ b/src/tracker/torrent.rs @@ -1,19 +1,16 @@ -use std::borrow::Cow; -use std::net::{IpAddr}; +use std::net::{IpAddr, SocketAddr}; use aquatic_udp_protocol::{AnnounceEvent}; use serde::{Deserialize, Serialize}; -use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; +use crate::{MAX_SCRAPE_TORRENTS, PeerId}; use crate::peer::TorrentPeer; #[derive(Serialize, Deserialize, Clone)] pub struct TorrentEntry { #[serde(skip)] - pub(crate) peers: std::collections::BTreeMap, - pub(crate) completed: u32, - #[serde(skip)] - pub(crate) seeders: u32, + pub peers: std::collections::BTreeMap, + pub completed: u32, } impl TorrentEntry { @@ -21,113 +18,53 @@ impl TorrentEntry { TorrentEntry { peers: std::collections::BTreeMap::new(), completed: 0, - seeders: 0, } } pub fn update_peer(&mut self, peer: &TorrentPeer) { match peer.event { AnnounceEvent::Stopped => { - let peer_old = self.peers.remove(&peer.peer_id); - self.update_torrent_stats_with_peer(peer, peer_old); + let _ = self.peers.remove(&peer.peer_id); } - _ => { + AnnounceEvent::Completed => { let peer_old = self.peers.insert(peer.peer_id.clone(), peer.clone()); - self.update_torrent_stats_with_peer(peer, peer_old); + // Don't count if peer was not previously known + if peer_old.is_some() { self.completed += 1; } + } + _ => { + let _ = self.peers.insert(peer.peer_id.clone(), peer.clone()); } } } - pub fn get_peers(&self, remote_addr: Option<&std::net::SocketAddr>) -> Vec { - let mut list = Vec::new(); - for (_, peer) in self - .peers - .iter() - .filter(|e| match remote_addr { - // don't filter on ip_version + pub fn get_peers(&self, client_addr: Option<&SocketAddr>) -> Vec<&TorrentPeer> { + self.peers + .values() + .filter(|peer| match client_addr { + // Don't filter on ip_version None => true, - // filter out different ip_version from remote_addr - Some(remote_address) => { - match e.1.peer_addr.ip() { - IpAddr::V4(_) => { remote_address.is_ipv4() } - IpAddr::V6(_) => { remote_address.is_ipv6() } + // Filter out different ip_version from remote_addr + Some(remote_addr) => { + // Skip ip address of client + if peer.peer_addr.ip() == remote_addr.ip() { return false; } + + match peer.peer_addr.ip() { + IpAddr::V4(_) => { remote_addr.is_ipv4() } + IpAddr::V6(_) => { remote_addr.is_ipv6() } } } }) .take(MAX_SCRAPE_TORRENTS as usize) - { - - // skip ip address of client - if let Some(remote_addr) = remote_addr { - if peer.peer_addr == *remote_addr { - continue; - } - } - - list.push(peer.clone()); - } - list - } - - pub fn update_torrent_stats_with_peer(&mut self, peer: &TorrentPeer, peer_old: Option) { - match peer_old { - None => { - if peer.is_seeder() { - self.seeders += 1; - } - - if peer.is_completed() { - self.completed += 1; - } - } - Some(peer_old) => { - match peer.event { - AnnounceEvent::None => { - if peer.is_seeder() && !peer_old.is_seeder() { - self.seeders += 1; - } - } - AnnounceEvent::Completed => { - if peer.is_seeder() && !peer_old.is_seeder() { - self.seeders += 1; - } - - // don't double count completed - if !peer_old.is_completed() { - self.completed += 1; - } - } - AnnounceEvent::Stopped => { - if peer_old.is_seeder() { - if self.seeders != 0 { - self.seeders -= 1; - } - } - } - // impossible, started should be the first time a peer announces itself - AnnounceEvent::Started => {} - } - } - } + .collect() } pub fn get_stats(&self) -> (u32, u32, u32) { - let leechers: u32 = if self.seeders < (self.peers.len() as u32) { - (self.peers.len() as u32) - self.seeders - } else { - 0 - }; - - (self.seeders, self.completed, leechers) + let seeders: u32 = self.peers.values().filter(|peer| peer.is_seeder()).count() as u32; + let leechers: u32 = self.peers.len() as u32 - seeders; + (seeders, self.completed, leechers) } } -#[derive(Serialize, Deserialize)] -struct DatabaseRow<'a> { - info_hash: InfoHash, - entry: Cow<'a, TorrentEntry>, -} - #[derive(Debug)] pub struct TorrentStats { pub completed: u32, diff --git a/src/tracker/tracker.rs b/src/tracker/tracker.rs index 4f0d571c6..d3e7c8faa 100644 --- a/src/tracker/tracker.rs +++ b/src/tracker/tracker.rs @@ -4,15 +4,14 @@ use std::net::SocketAddr; use std::sync::Arc; use log::info; -use serde::{Deserialize, Serialize}; -use serde; use tokio::sync::{RwLock, RwLockReadGuard}; +use tokio::sync::mpsc::error::SendError; use crate::Configuration; use crate::protocol::common::InfoHash; use crate::databases::database::Database; -use tokio::sync::mpsc::error::SendError; use crate::databases::database; +use crate::mode::TrackerMode; use crate::peer::TorrentPeer; use crate::tracker::key::AuthKey; use crate::tracker::key::Error::KeyInvalid; @@ -20,28 +19,9 @@ use crate::statistics::{StatsTracker, TrackerStatistics, TrackerStatisticsEvent} use crate::tracker::key; use crate::tracker::torrent::{TorrentEntry, TorrentError, TorrentStats}; -#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Debug)] -pub enum TrackerMode { - // Will track every new info hash and serve every peer. - #[serde(rename = "public")] - PublicMode, - - // Will only track whitelisted info hashes. - #[serde(rename = "listed")] - ListedMode, - - // Will only serve authenticated peers - #[serde(rename = "private")] - PrivateMode, - - // Will only track whitelisted info hashes and serve authenticated peers - #[serde(rename = "private_listed")] - PrivateListedMode, -} - pub struct TorrentTracker { - mode: TrackerMode, pub config: Arc, + mode: TrackerMode, torrents: RwLock>, updates: RwLock>, shadow: RwLock>, @@ -58,8 +38,8 @@ impl TorrentTracker { if config.statistics { stats_tracker.run_worker(); } Ok(TorrentTracker { - mode: config.mode, config: config.clone(), + mode: config.mode, torrents: RwLock::new(std::collections::BTreeMap::new()), updates: RwLock::new(std::collections::HashMap::new()), shadow: RwLock::new(std::collections::HashMap::new()), @@ -69,15 +49,15 @@ impl TorrentTracker { } pub fn is_public(&self) -> bool { - self.mode == TrackerMode::PublicMode + self.mode == TrackerMode::Public } pub fn is_private(&self) -> bool { - self.mode == TrackerMode::PrivateMode || self.mode == TrackerMode::PrivateListedMode + self.mode == TrackerMode::Private || self.mode == TrackerMode::PrivateListed } pub fn is_whitelisted(&self) -> bool { - self.mode == TrackerMode::ListedMode || self.mode == TrackerMode::PrivateListedMode + self.mode == TrackerMode::Listed || self.mode == TrackerMode::PrivateListed } pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { @@ -138,7 +118,6 @@ impl TorrentTracker { let torrent_entry = TorrentEntry { peers: Default::default(), completed, - seeders: Default::default(), }; torrents.insert(info_hash.clone(), torrent_entry); @@ -171,16 +150,13 @@ impl TorrentTracker { } - pub async fn get_torrent_peers( - &self, - info_hash: &InfoHash, - peer_addr: &SocketAddr, - ) -> Vec { + pub async fn get_torrent_peers(&self, info_hash: &InfoHash, client_addr: &SocketAddr, ) -> Vec { let read_lock = self.torrents.read().await; + match read_lock.get(info_hash) { None => vec![], Some(entry) => { - entry.get_peers(Some(peer_addr)) + entry.get_peers(Some(client_addr)).into_iter().cloned().collect() } } } @@ -242,6 +218,7 @@ impl TorrentTracker { info!("-=[ Stats ]=- | Torrents: {} | Updates: {} | Shadow: {}", torrents_size, updates_size, shadow_size); } + // todo: refactor // remove torrents without peers if enabled, and defragment memory pub async fn cleanup_torrents(&self) { let lock = self.torrents.write().await; @@ -260,7 +237,6 @@ impl TorrentTracker { let mut torrent = TorrentEntry { peers: BTreeMap::new(), completed: 0, - seeders: 0, }; let lock = self.torrents.write().await; @@ -273,13 +249,10 @@ impl TorrentTracker { continue; } torrent.peers.insert(peer_id.clone(), peer.clone()); - if peer.is_seeder() { - torrent.seeders += 1; - } } let mut lock = self.torrents.write().await; lock.remove(hash); - if self.config.mode.clone() == TrackerMode::PublicMode && self.config.cleanup_peerless && !self.config.persistence { + if self.config.mode.clone() == TrackerMode::Public && self.config.cleanup_peerless && !self.config.persistence { if torrent.peers.len() != 0 { lock.insert(hash.clone(), torrent); } @@ -290,6 +263,7 @@ impl TorrentTracker { } } + // todo: refactor pub async fn periodic_saving(&self) { // Get a lock for writing // let mut shadow = self.shadow.write().await; @@ -298,14 +272,14 @@ impl TorrentTracker { let mut updates = self.updates.write().await; let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); - info!("Copying updates to updates_cloned..."); + // Copying updates to updates_cloned for (k, completed) in updates.iter() { updates_cloned.insert(k.clone(), completed.clone()); } updates.clear(); drop(updates); - info!("Copying updates_cloned into the shadow to overwrite..."); + // Copying updates_cloned into the shadow to overwrite for (k, completed) in updates_cloned.iter() { let mut shadows = self.shadow.write().await; if shadows.contains_key(k) { @@ -317,14 +291,13 @@ impl TorrentTracker { drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. - info!("Handle shadow_copy to be updated into SQL..."); + // Handle shadow_copy to be updated into SQL let mut shadow_copy: BTreeMap = BTreeMap::new(); let shadows = self.shadow.read().await; for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), completed: completed.clone(), - seeders: 0, }); } drop(shadows); From 6fcc043c0e38091b55add786f4e9c33f68a0ed73 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 01:49:16 +0200 Subject: [PATCH 39/53] refactor: removed unused enum in config --- src/config.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/config.rs b/src/config.rs index c24d1c45d..729406163 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,12 +12,6 @@ use toml; use crate::databases::database::DatabaseDrivers; use crate::mode::TrackerMode; -#[derive(Serialize, Deserialize, PartialEq)] -pub enum TrackerServer { - UDP, - HTTP, -} - #[derive(Serialize, Deserialize, Debug)] pub struct UdpTrackerConfig { pub enabled: bool, From abe2e685444aa0ba2925dfe55668fdca2f519af5 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 02:02:04 +0200 Subject: [PATCH 40/53] refactor: removed unused byteorder dependency --- Cargo.lock | 206 ++++++++++++++++++++++-------------------- Cargo.toml | 19 ++-- src/protocol/utils.rs | 27 ------ 3 files changed, 120 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56ecd77a7..4d6b9b17a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,7 @@ dependencies = [ [[package]] name = "aquatic_udp_protocol" version = "0.2.0" -source = "git+https://github.com/greatest-ape/aquatic#26e2e874377a2682f52568f8e5e8c080c3366326" +source = "git+https://github.com/greatest-ape/aquatic#99792eefc3a0cfb15dc9bbd351af94b14a44e9fc" dependencies = [ "byteorder", "either", @@ -88,9 +88,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base-x" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +checksum = "dc19a4937b4fbd3fe3379793130e42060d10627a360f2127802b10b87e7baf74" [[package]] name = "base64" @@ -106,8 +106,8 @@ checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256" dependencies = [ "num-bigint 0.3.3", "num-integer", - "num-traits 0.2.14", - "serde 1.0.136", + "num-traits 0.2.15", + "serde 1.0.137", ] [[package]] @@ -238,8 +238,8 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ "libc", "num-integer", - "num-traits 0.2.14", - "serde 1.0.136", + "num-traits 0.2.15", + "serde 1.0.137", "time 0.1.44", "winapi", ] @@ -288,7 +288,7 @@ dependencies = [ "lazy_static", "nom", "rust-ini", - "serde 1.0.136", + "serde 1.0.137", "serde-hjson", "serde_json", "toml", @@ -442,9 +442,9 @@ dependencies = [ [[package]] name = "fern" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9a4820f0ccc8a7afd67c39a0f1a0f4b07ca1725164271a64939d7aeb9af065" +checksum = "3bdd7b0849075e79ee9a1836df22c717d1eba30451796fdc631b04565dd11e2a" dependencies = [ "log", ] @@ -749,9 +749,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", @@ -771,9 +771,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -905,9 +905,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" [[package]] name = "libloading" @@ -931,9 +931,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" dependencies = [ "cc", "pkg-config", @@ -958,9 +958,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] @@ -991,9 +991,9 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -1088,7 +1088,7 @@ dependencies = [ "once_cell", "pem", "percent-encoding", - "serde 1.0.136", + "serde 1.0.137", "serde_json", "socket2", "twox-hash", @@ -1117,12 +1117,12 @@ dependencies = [ "lazy_static", "lexical", "num-bigint 0.4.3", - "num-traits 0.2.14", + "num-traits 0.2.15", "rand", "regex", "rust_decimal", "saturating", - "serde 1.0.136", + "serde 1.0.137", "serde_json", "sha1", "sha2", @@ -1201,7 +1201,7 @@ checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] @@ -1212,17 +1212,17 @@ checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ "autocfg", "num-integer", - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] @@ -1231,14 +1231,14 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -1267,18 +1267,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1287,9 +1299,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -1316,7 +1328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.2", + "parking_lot_core 0.9.3", ] [[package]] @@ -1335,9 +1347,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -1391,9 +1403,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1421,9 +1433,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" dependencies = [ "unicode-xid", ] @@ -1589,8 +1601,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22dc69eadbf0ee2110b8d20418c0c6edbaefec2811c4963dc17b6344e11fe0f8" dependencies = [ "arrayvec 0.7.2", - "num-traits 0.2.14", - "serde 1.0.136", + "num-traits 0.2.15", + "serde 1.0.137", ] [[package]] @@ -1614,7 +1626,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.7", + "semver 1.0.9", ] [[package]] @@ -1723,9 +1735,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" [[package]] name = "semver-parser" @@ -1741,9 +1753,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] @@ -1766,24 +1778,24 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "934d8bdbaa0126dafaea9a8833424a211d9661897717846c6bb782349ca1c30d" dependencies = [ - "serde 1.0.136", + "serde 1.0.137", "serde_bytes", ] [[package]] name = "serde_bytes" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" +checksum = "212e73464ebcde48d723aa02eb270ba62eff38a9b732df31f33f1b4e145f3a54" dependencies = [ - "serde 1.0.136", + "serde 1.0.137", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -1792,13 +1804,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", - "serde 1.0.136", + "serde 1.0.137", ] [[package]] @@ -1810,7 +1822,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.136", + "serde 1.0.137", ] [[package]] @@ -1945,7 +1957,7 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", - "serde 1.0.136", + "serde 1.0.137", "serde_derive", "syn", ] @@ -1959,7 +1971,7 @@ dependencies = [ "base-x", "proc-macro2", "quote", - "serde 1.0.136", + "serde 1.0.137", "serde_derive", "serde_json", "sha1", @@ -1990,9 +2002,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" dependencies = [ "proc-macro2", "quote", @@ -2039,18 +2051,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -2108,9 +2120,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -2123,9 +2135,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" dependencies = [ "bytes", "libc", @@ -2221,7 +2233,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ - "serde 1.0.136", + "serde 1.0.137", ] [[package]] @@ -2231,7 +2243,6 @@ dependencies = [ "aquatic_udp_protocol", "async-trait", "binascii", - "byteorder", "chrono", "config", "derive_more", @@ -2244,9 +2255,8 @@ dependencies = [ "r2d2_mysql", "r2d2_sqlite", "rand", - "serde 1.0.136", + "serde 1.0.137", "serde_bencode", - "serde_bytes", "serde_json", "thiserror", "tokio", @@ -2275,9 +2285,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -2329,9 +2339,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "1.6.2" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "rand", @@ -2355,9 +2365,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-normalization" @@ -2376,9 +2386,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "untrusted" @@ -2457,7 +2467,7 @@ dependencies = [ "percent-encoding", "pin-project", "scoped-tls", - "serde 1.0.136", + "serde 1.0.137", "serde_json", "serde_urlencoded", "tokio", @@ -2597,9 +2607,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2610,33 +2620,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "wyz" diff --git a/Cargo.toml b/Cargo.toml index cc97072a5..420e772fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,27 +11,32 @@ debug = 1 lto = "fat" [dependencies] +tokio = { version = "1.7", features = ["full"] } + serde = { version = "1.0", features = ["derive"] } serde_bencode = "^0.2.3" -serde_bytes = "0.11" serde_json = "1.0.72" hex = "0.4.3" percent-encoding = "2.1.0" -warp = { version = "0.3", features = ["tls"] } -tokio = { version = "1.7", features = ["full"] } binascii = "0.1" + +warp = { version = "0.3", features = ["tls"] } + +config = "0.11" toml = "0.5" + log = { version = "0.4", features = ["release_max_level_info"] } fern = "0.6" chrono = "0.4" -byteorder = "1" + +r2d2 = "0.8.8" r2d2_mysql = "21.0.0" r2d2_sqlite = "0.16.0" -r2d2 = "0.8.8" + rand = "0.8.4" -config = "0.11" derive_more = "0.99" thiserror = "1.0" -aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } futures = "0.3.21" async-trait = "0.1.52" + +aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } diff --git a/src/protocol/utils.rs b/src/protocol/utils.rs index 5fc694c8e..392966307 100644 --- a/src/protocol/utils.rs +++ b/src/protocol/utils.rs @@ -1,11 +1,9 @@ use std::error::Error; use std::fmt::Write; -use std::io::Cursor; use std::net::SocketAddr; use std::time::SystemTime; use aquatic_udp_protocol::ConnectionId; -use byteorder::{BigEndian, ReadBytesExt}; pub fn get_connection_id(remote_address: &SocketAddr) -> ConnectionId { match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) { @@ -33,31 +31,6 @@ pub fn url_encode_bytes(content: &[u8]) -> Result> { Ok(out) } -// Function that will convert a small or big number into the smallest form of a byte array. -pub async fn convert_int_to_bytes(number: &u64) -> Vec { - let mut return_data: Vec = Vec::new(); - // return_data.extend(number.to_be_bytes().reverse()); - for i in 1..8 { - if number < &256u64.pow(i) { - let start: usize = 16usize - i as usize; - return_data.extend(number.to_be_bytes()[start..8].iter()); - return return_data; - } - } - return return_data; -} - -pub async fn convert_bytes_to_int(array: &Vec) -> u64 { - let mut array_fixed: Vec = Vec::new(); - let size = 8 - array.len(); - for _ in 0..size { - array_fixed.push(0); - } - array_fixed.extend(array); - let mut rdr = Cursor::new(array_fixed); - return rdr.read_u64::().unwrap(); -} - pub fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { ser.serialize_u64(inst.elapsed().as_millis() as u64) } From d6dd240c080363daedcad067fb1910c8ae31282d Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 02:05:01 +0200 Subject: [PATCH 41/53] refactor: updated rust version to 2021 --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 420e772fd..a98820521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,11 @@ [package] +edition = "2021" name = "torrust-tracker" version = "2.3.0" license = "AGPL-3.0" authors = ["Mick van Dijke "] description = "A feature rich BitTorrent tracker." -edition = "2018" +repository = "https://github.com/torrust/torrust-tracker" [profile.release] debug = 1 From 278ac3abebf2f8c0bebe8faf7d9d00067da808f8 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 14:11:29 +0200 Subject: [PATCH 42/53] refactor: updated cargo.toml dev and release profiles --- Cargo.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index a98820521..4c2fa4a86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,14 @@ authors = ["Mick van Dijke "] description = "A feature rich BitTorrent tracker." repository = "https://github.com/torrust/torrust-tracker" +[profile.dev] +debug = 1 +opt-level = 1 +lto = "thin" + [profile.release] debug = 1 +opt-level = 3 lto = "fat" [dependencies] From 829afb9606d311022892898804d947288e86af49 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 17:49:03 +0200 Subject: [PATCH 43/53] refactor: refactored and improved tracker.cleanup_torrents() --- src/config.rs | 16 +++---- src/jobs/persistent_torrent_statistics.rs | 2 +- src/jobs/torrent_cleanup.rs | 2 +- src/setup.rs | 5 ++- src/tracker/torrent.rs | 6 +++ src/tracker/tracker.rs | 52 ++++++----------------- 6 files changed, 33 insertions(+), 50 deletions(-) diff --git a/src/config.rs b/src/config.rs index 729406163..292b065b4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -53,10 +53,10 @@ pub struct Configuration { pub db_driver: DatabaseDrivers, pub db_path: String, pub statistics: bool, - pub persistence: bool, - pub persistence_interval: Option, - pub cleanup_interval: u64, - pub cleanup_peerless: bool, + pub persistent_torrent_completed_stat: bool, + pub persistence_interval: u64, + pub inactive_peer_cleanup_interval: u64, + pub remove_peerless_torrents: bool, pub external_ip: Option, pub announce_interval: u32, pub announce_interval_min: u32, @@ -139,10 +139,10 @@ impl Configuration { db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), statistics: true, - persistence: false, - persistence_interval: Some(900), - cleanup_interval: 600, - cleanup_peerless: true, + persistent_torrent_completed_stat: false, + persistence_interval: 0, + inactive_peer_cleanup_interval: 600, + remove_peerless_torrents: true, external_ip: Some(String::from("0.0.0.0")), announce_interval: 120, announce_interval_min: 120, diff --git a/src/jobs/persistent_torrent_statistics.rs b/src/jobs/persistent_torrent_statistics.rs index 7ebc80bdb..54ee23b6b 100644 --- a/src/jobs/persistent_torrent_statistics.rs +++ b/src/jobs/persistent_torrent_statistics.rs @@ -6,7 +6,7 @@ use crate::tracker::tracker::TorrentTracker; pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.persistence_interval.unwrap_or(900); + let interval = config.persistence_interval; tokio::spawn(async move { let interval = std::time::Duration::from_secs(interval); diff --git a/src/jobs/torrent_cleanup.rs b/src/jobs/torrent_cleanup.rs index 05e639728..7d9967352 100644 --- a/src/jobs/torrent_cleanup.rs +++ b/src/jobs/torrent_cleanup.rs @@ -7,7 +7,7 @@ use crate::tracker::tracker::TorrentTracker; pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.cleanup_interval; + let interval = config.inactive_peer_cleanup_interval; tokio::spawn(async move { let interval = std::time::Duration::from_secs(interval); diff --git a/src/setup.rs b/src/setup.rs index b8d49614d..76372ef0d 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -8,8 +8,9 @@ use crate::tracker::tracker::TorrentTracker; pub async fn setup(config: &Configuration, tracker: Arc) -> Vec>{ let mut jobs: Vec> = Vec::new(); + // todo: replace by realtime updates // Load persistent torrents - if config.persistence { + if config.persistent_torrent_completed_stat && config.persistence_interval > 0 { info!("Loading persistent torrents into memory.."); tracker.load_persistent_torrents().await.expect("Could not load persistent torrents."); info!("Persistent torrents loaded."); @@ -39,7 +40,7 @@ pub async fn setup(config: &Configuration, tracker: Arc) -> Vec< } // Remove torrents without peers, every interval - if config.cleanup_interval > 0 { + if config.inactive_peer_cleanup_interval > 0 { jobs.push(torrent_cleanup::start_job(&config, tracker.clone())); } diff --git a/src/tracker/torrent.rs b/src/tracker/torrent.rs index a01b5ce55..4b891e992 100644 --- a/src/tracker/torrent.rs +++ b/src/tracker/torrent.rs @@ -63,6 +63,12 @@ impl TorrentEntry { let leechers: u32 = self.peers.len() as u32 - seeders; (seeders, self.completed, leechers) } + + pub fn remove_inactive_peers(&mut self, max_peer_timeout: u32) { + self.peers.retain(|_, peer| { + peer.updated.elapsed() > std::time::Duration::from_secs(max_peer_timeout as u64) + }); + } } #[derive(Debug)] diff --git a/src/tracker/tracker.rs b/src/tracker/tracker.rs index d3e7c8faa..d25f32923 100644 --- a/src/tracker/tracker.rs +++ b/src/tracker/tracker.rs @@ -177,7 +177,7 @@ impl TorrentTracker { let (seeders, completed, leechers) = torrent_entry.get_stats(); - if self.config.persistence { + if self.config.persistent_torrent_completed_stat { let mut updates = self.updates.write().await; if updates.contains_key(info_hash) { updates.remove(info_hash); @@ -218,48 +218,24 @@ impl TorrentTracker { info!("-=[ Stats ]=- | Torrents: {} | Updates: {} | Shadow: {}", torrents_size, updates_size, shadow_size); } - // todo: refactor - // remove torrents without peers if enabled, and defragment memory + // Remove inactive peers and (optionally) peerless torrents pub async fn cleanup_torrents(&self) { - let lock = self.torrents.write().await; - - // First we create a mapping of all the torrent hashes in a vector, and we use this to iterate through the btreemap. - // Every hash we have handled, we remove from the btreemap completely, and push it to the top. - let mut torrent_hashes: Vec = Vec::new(); - for (k, _torrent_entry) in lock.iter() { - torrent_hashes.push(k.clone()); - } - - drop(lock); - - // Let's iterate through all torrents, and parse. - for hash in torrent_hashes.iter() { - let mut torrent = TorrentEntry { - peers: BTreeMap::new(), - completed: 0, - }; + let mut torrents_lock = self.torrents.write().await; - let lock = self.torrents.write().await; - let torrent_data = lock.get(hash).unwrap().clone(); - drop(lock); + // If we don't need to remove torrents we will use the faster iter + if self.config.remove_peerless_torrents { + torrents_lock.retain(|_, torrent_entry| { + torrent_entry.remove_inactive_peers(self.config.peer_timeout); - torrent.completed = torrent_data.completed.clone(); - for (peer_id, peer) in torrent_data.peers.iter() { - if peer.updated.elapsed() > std::time::Duration::from_secs(self.config.peer_timeout as u64) { - continue; - } - torrent.peers.insert(peer_id.clone(), peer.clone()); - } - let mut lock = self.torrents.write().await; - lock.remove(hash); - if self.config.mode.clone() == TrackerMode::Public && self.config.cleanup_peerless && !self.config.persistence { - if torrent.peers.len() != 0 { - lock.insert(hash.clone(), torrent); + match self.config.persistent_torrent_completed_stat { + true => { torrent_entry.completed > 0 || torrent_entry.peers.len() > 0 } + false => { torrent_entry.peers.len() > 0 } } - } else { - lock.insert(hash.clone(), torrent); + }); + } else { + for (_, torrent_entry) in torrents_lock.iter_mut() { + torrent_entry.remove_inactive_peers(self.config.peer_timeout); } - drop(lock); } } From eb69fa00595989b23cdf71f9e4c954669ff018ac Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 18:08:22 +0200 Subject: [PATCH 44/53] feat: added strip to Cargo.toml to almost half compiled binary size on unix systems --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 4c2fa4a86..c320e9e03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ lto = "thin" debug = 1 opt-level = 3 lto = "fat" +strip = true [dependencies] tokio = { version = "1.7", features = ["full"] } From 1d353cadb337b7ab847d47f5b903d931edabc2b8 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 7 May 2022 23:47:08 +0200 Subject: [PATCH 45/53] feat: massively improved key authentication speed in private mode --- src/api/server.rs | 2 +- src/databases/database.rs | 4 +++- src/databases/mysql.rs | 17 +++++++++++++++-- src/databases/sqlite.rs | 24 ++++++++++++++++++++++-- src/setup.rs | 4 ++++ src/tracker/tracker.rs | 38 ++++++++++++++++++++++++++++++++------ 6 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/api/server.rs b/src/api/server.rs index d33f17dc0..77496b497 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -289,7 +289,7 @@ pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp }) .and_then(|(key, tracker): (String, Arc)| { async move { - match tracker.remove_auth_key(key).await { + match tracker.remove_auth_key(&key).await { Ok(_) => Ok(warp::reply::json(&ActionStatus::Ok)), Err(_) => Err(warp::reject::custom(ActionStatus::Err { reason: "failed to delete key".into() })) } diff --git a/src/databases/database.rs b/src/databases/database.rs index fd9f2a19d..e2f3cdfe5 100644 --- a/src/databases/database.rs +++ b/src/databases/database.rs @@ -42,6 +42,8 @@ pub trait Database: Sync + Send { async fn load_persistent_torrents(&self) -> Result, Error>; + async fn load_keys(&self) -> Result, Error>; + async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), Error>; async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result; @@ -54,7 +56,7 @@ pub trait Database: Sync + Send { async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result; - async fn remove_key_from_keys(&self, key: String) -> Result; + async fn remove_key_from_keys(&self, key: &str) -> Result; } #[derive(Debug, Display, PartialEq, Error)] diff --git a/src/databases/mysql.rs b/src/databases/mysql.rs index e7f57a7a4..6fd63b7d9 100644 --- a/src/databases/mysql.rs +++ b/src/databases/mysql.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::str::FromStr; use async_trait::async_trait; -use log::debug; +use log::{debug}; use r2d2::Pool; use r2d2_mysql::mysql::{Opts, OptsBuilder, params, TxOpts}; use r2d2_mysql::mysql::prelude::Queryable; @@ -76,6 +76,19 @@ impl Database for MysqlDatabase { Ok(torrents) } + async fn load_keys(&self) -> Result, Error> { + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; + + let keys: Vec = conn.query_map("SELECT `key`, valid_until FROM `keys`", |(key, valid_until): (String, i64)| { + AuthKey { + key, + valid_until: Some(valid_until as u64) + } + }).map_err(|_| database::Error::QueryReturnedNoRows)?; + + Ok(keys) + } + async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), database::Error> { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; @@ -190,7 +203,7 @@ impl Database for MysqlDatabase { } } - async fn remove_key_from_keys(&self, key: String) -> Result { + async fn remove_key_from_keys(&self, key: &str) -> Result { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; match conn.exec_drop("DELETE FROM `keys` WHERE key = :key", params! { key }) { diff --git a/src/databases/sqlite.rs b/src/databases/sqlite.rs index 18a1d5a28..9c452eadc 100644 --- a/src/databases/sqlite.rs +++ b/src/databases/sqlite.rs @@ -8,7 +8,7 @@ use r2d2_sqlite::SqliteConnectionManager; use r2d2_sqlite::rusqlite::NO_PARAMS; use crate::{AUTH_KEY_LENGTH, InfoHash}; -use crate::databases::database::Database; +use crate::databases::database::{Database, Error}; use crate::databases::database; use crate::tracker::key::AuthKey; use crate::tracker::torrent::TorrentEntry; @@ -76,6 +76,26 @@ impl Database for SqliteDatabase { Ok(torrents) } + async fn load_keys(&self) -> Result, Error> { + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; + + let mut stmt = conn.prepare("SELECT key, valid_until FROM keys")?; + + let keys_iter = stmt.query_map(NO_PARAMS, |row| { + let key = row.get(0)?; + let valid_until: i64 = row.get(1)?; + + Ok(AuthKey { + key, + valid_until: Some(valid_until as u64) + }) + })?; + + let keys: Vec = keys_iter.filter_map(|x| x.ok()).collect(); + + Ok(keys) + } + async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), database::Error> { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; @@ -174,7 +194,7 @@ impl Database for SqliteDatabase { } } - async fn remove_key_from_keys(&self, key: String) -> Result { + async fn remove_key_from_keys(&self, key: &str) -> Result { let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; match conn.execute("DELETE FROM keys WHERE key = ?", &[key]) { diff --git a/src/setup.rs b/src/setup.rs index 76372ef0d..69f2db432 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -8,6 +8,10 @@ use crate::tracker::tracker::TorrentTracker; pub async fn setup(config: &Configuration, tracker: Arc) -> Vec>{ let mut jobs: Vec> = Vec::new(); + if tracker.is_private() { + tracker.load_keys().await.expect("Could not retrieve keys."); + } + // todo: replace by realtime updates // Load persistent torrents if config.persistent_torrent_completed_stat && config.persistence_interval > 0 { diff --git a/src/tracker/tracker.rs b/src/tracker/tracker.rs index d25f32923..fdbaac4de 100644 --- a/src/tracker/tracker.rs +++ b/src/tracker/tracker.rs @@ -14,7 +14,6 @@ use crate::databases::database; use crate::mode::TrackerMode; use crate::peer::TorrentPeer; use crate::tracker::key::AuthKey; -use crate::tracker::key::Error::KeyInvalid; use crate::statistics::{StatsTracker, TrackerStatistics, TrackerStatisticsEvent}; use crate::tracker::key; use crate::tracker::torrent::{TorrentEntry, TorrentError, TorrentStats}; @@ -22,6 +21,7 @@ use crate::tracker::torrent::{TorrentEntry, TorrentError, TorrentStats}; pub struct TorrentTracker { pub config: Arc, mode: TrackerMode, + keys: RwLock>, torrents: RwLock>, updates: RwLock>, shadow: RwLock>, @@ -40,6 +40,7 @@ impl TorrentTracker { Ok(TorrentTracker { config: config.clone(), mode: config.mode, + keys: RwLock::new(std::collections::HashMap::new()), torrents: RwLock::new(std::collections::BTreeMap::new()), updates: RwLock::new(std::collections::HashMap::new()), shadow: RwLock::new(std::collections::HashMap::new()), @@ -66,18 +67,32 @@ impl TorrentTracker { // add key to database if let Err(error) = self.database.add_key_to_keys(&auth_key).await { return Err(error); } + // Add key to in-memory database + self.keys.write().await.insert(auth_key.key.clone(), auth_key.clone()); + Ok(auth_key) } - pub async fn remove_auth_key(&self, key: String) -> Result { - self.database.remove_key_from_keys(key).await + pub async fn remove_auth_key(&self, key: &str) -> Result { + self.database.remove_key_from_keys(&key).await?; + + // Remove key from in-memory database + self.keys.write().await.remove(key); + + Ok(1) } pub async fn verify_auth_key(&self, auth_key: &AuthKey) -> Result<(), key::Error> { - let db_key = self.database.get_key_from_keys(&auth_key.key).await.map_err(|_| KeyInvalid)?; - key::verify_auth_key(&db_key) + let keys_lock = self.keys.read().await; + + if let Some(key) = keys_lock.get(&auth_key.key) { + key::verify_auth_key(key) + } else { + Err(key::Error::KeyInvalid) + } } + // todo: speed this up in non-public modes pub async fn authenticate_request(&self, info_hash: &InfoHash, key: &Option) -> Result<(), TorrentError> { // no authentication needed in public mode if self.is_public() { return Ok(()); } @@ -98,7 +113,7 @@ impl TorrentTracker { // check if info_hash is whitelisted if self.is_whitelisted() { - if self.is_info_hash_whitelisted(info_hash).await == false { + if !self.is_info_hash_whitelisted(info_hash).await { return Err(TorrentError::TorrentNotWhitelisted); } } @@ -106,6 +121,17 @@ impl TorrentTracker { Ok(()) } + pub async fn load_keys(&self) -> Result<(), database::Error> { + let keys_from_database = self.database.load_keys().await?; + let mut keys = self.keys.write().await; + + for key in keys_from_database { + let _ = keys.insert(key.key.clone(), key); + } + + Ok(()) + } + // Loading the torrents from database into memory pub async fn load_persistent_torrents(&self) -> Result<(), database::Error> { let persistent_torrents = self.database.load_persistent_torrents().await?; From aac9ac72bdd300ffb2959ba6f2a1ee04ecac5895 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sun, 8 May 2022 17:26:44 +0200 Subject: [PATCH 46/53] fix: http scrape response --- src/http/filters.rs | 2 ++ src/http/handlers.rs | 16 +++++++++------- src/http/response.rs | 25 ++++++++++++++++++++++--- src/protocol/utils.rs | 13 ------------- src/tracker/tracker.rs | 2 +- 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/http/filters.rs b/src/http/filters.rs index 8f3ee04c0..d5a7881d8 100644 --- a/src/http/filters.rs +++ b/src/http/filters.rs @@ -75,9 +75,11 @@ async fn info_hashes(raw_query: String) -> WebResult> { for v in split_raw_query { if v.contains("info_hash") { let raw_info_hash = v.split("=").collect::>()[1]; + debug!("Raw info hash: {}", raw_info_hash); let info_hash_bytes = percent_encoding::percent_decode_str(raw_info_hash).collect::>(); let info_hash = InfoHash::from_str(&hex::encode(info_hash_bytes)); if let Ok(ih) = info_hash { + debug!("Parsed info hash: {}", ih.to_string()); info_hashes.push(ih); } } diff --git a/src/http/handlers.rs b/src/http/handlers.rs index d7e4859d9..977aeb4ba 100644 --- a/src/http/handlers.rs +++ b/src/http/handlers.rs @@ -13,7 +13,6 @@ use crate::tracker::torrent::{TorrentError, TorrentStats}; use crate::http::{AnnounceRequest, AnnounceResponse, ErrorResponse, Peer, ScrapeRequest, ScrapeResponse, ScrapeResponseEntry, ServerError, WebResult}; use crate::peer::TorrentPeer; use crate::tracker::statistics::TrackerStatisticsEvent; -use crate::protocol::utils::url_encode_bytes; use crate::tracker::tracker::TorrentTracker; /// Authenticate InfoHash using optional AuthKey @@ -62,7 +61,7 @@ pub async fn handle_announce(announce_request: AnnounceRequest, auth_key: Option /// Handle scrape request pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option, tracker: Arc) -> WebResult { - let mut files: HashMap = HashMap::new(); + let mut files: HashMap = HashMap::new(); let db = tracker.get_torrents().await; for info_hash in scrape_request.info_hashes.iter() { @@ -79,9 +78,7 @@ pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option) -> WebResult { - Ok(Response::new(ScrapeResponse { files }.write())) +fn send_scrape_response(files: HashMap) -> WebResult { + let res = ScrapeResponse { files }; + + match res.write() { + Ok(body) => Ok(Response::new(body)), + Err(_) => Err(reject::custom(ServerError::InternalServerError)) + } } /// Handle all server errors and send error reply diff --git a/src/http/response.rs b/src/http/response.rs index 3118f7df1..2bdd4c1e7 100644 --- a/src/http/response.rs +++ b/src/http/response.rs @@ -5,6 +5,7 @@ use std::net::IpAddr; use serde; use serde::Serialize; +use crate::InfoHash; #[derive(Serialize)] pub struct Peer { @@ -78,12 +79,30 @@ pub struct ScrapeResponseEntry { #[derive(Serialize)] pub struct ScrapeResponse { - pub files: HashMap, + pub files: HashMap, } impl ScrapeResponse { - pub fn write(&self) -> String { - serde_bencode::to_string(&self).unwrap() + pub fn write(&self) -> Result, Box> { + let mut bytes: Vec = Vec::new(); + + bytes.write(b"d5:filesd")?; + + for (info_hash, scrape_response_entry) in self.files.iter() { + bytes.write(b"20:")?; + bytes.write(&info_hash.0)?; + bytes.write(b"d8:completei")?; + bytes.write(scrape_response_entry.complete.to_string().as_bytes())?; + bytes.write(b"e10:downloadedi")?; + bytes.write(scrape_response_entry.downloaded.to_string().as_bytes())?; + bytes.write(b"e10:incompletei")?; + bytes.write(scrape_response_entry.incomplete.to_string().as_bytes())?; + bytes.write(b"ee")?; + } + + bytes.write(b"ee")?; + + Ok(bytes) } } diff --git a/src/protocol/utils.rs b/src/protocol/utils.rs index 392966307..2b8ac3ebf 100644 --- a/src/protocol/utils.rs +++ b/src/protocol/utils.rs @@ -18,19 +18,6 @@ pub fn current_time() -> u64 { .as_secs() } -pub fn url_encode_bytes(content: &[u8]) -> Result> { - let mut out: String = String::new(); - - for byte in content.iter() { - match *byte as char { - '0'..='9' | 'a'..='z' | 'A'..='Z' | '.' | '-' | '_' | '~' => out.push(*byte as char), - _ => write!(&mut out, "%{:02x}", byte)?, - }; - } - - Ok(out) -} - pub fn ser_instant(inst: &std::time::Instant, ser: S) -> Result { ser.serialize_u64(inst.elapsed().as_millis() as u64) } diff --git a/src/tracker/tracker.rs b/src/tracker/tracker.rs index fdbaac4de..45b23f728 100644 --- a/src/tracker/tracker.rs +++ b/src/tracker/tracker.rs @@ -92,7 +92,6 @@ impl TorrentTracker { } } - // todo: speed this up in non-public modes pub async fn authenticate_request(&self, info_hash: &InfoHash, key: &Option) -> Result<(), TorrentError> { // no authentication needed in public mode if self.is_public() { return Ok(()); } @@ -111,6 +110,7 @@ impl TorrentTracker { } } + // todo: speed this up // check if info_hash is whitelisted if self.is_whitelisted() { if !self.is_info_hash_whitelisted(info_hash).await { From c4c6c3ddcb31c580cdc133eaa76191144a84ce58 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 18:28:02 +0200 Subject: [PATCH 47/53] feat: improved persistent torrent saving --- Cargo.lock | 3 +- Cargo.toml | 2 +- src/config.rs | 37 +++---- src/databases/database.rs | 10 +- src/databases/mysql.rs | 62 +++++------- src/databases/sqlite.rs | 37 ++++--- src/http/filters.rs | 6 +- src/http/handlers.rs | 2 +- src/jobs/log_statistics.rs | 32 ------ src/jobs/mod.rs | 2 - src/jobs/persistent_torrent_statistics.rs | 38 ------- src/protocol/utils.rs | 2 - src/setup.rs | 24 ++--- src/tracker/statistics.rs | 24 +++-- src/tracker/torrent.rs | 12 ++- src/tracker/tracker.rs | 118 +++++----------------- 16 files changed, 134 insertions(+), 277 deletions(-) delete mode 100644 src/jobs/log_statistics.rs delete mode 100644 src/jobs/persistent_torrent_statistics.rs diff --git a/Cargo.lock b/Cargo.lock index 4d6b9b17a..1ee924fac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,8 @@ dependencies = [ [[package]] name = "aquatic_udp_protocol" version = "0.2.0" -source = "git+https://github.com/greatest-ape/aquatic#99792eefc3a0cfb15dc9bbd351af94b14a44e9fc" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16149f27924d42b337a637cd90a8ee2a8973bbccf32aabebce2b3c66913f947f" dependencies = [ "byteorder", "either", diff --git a/Cargo.toml b/Cargo.toml index c320e9e03..53e2949db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,4 +47,4 @@ thiserror = "1.0" futures = "0.3.21" async-trait = "0.1.52" -aquatic_udp_protocol = { git = "https://github.com/greatest-ape/aquatic" } +aquatic_udp_protocol = "0.2.0" diff --git a/src/config.rs b/src/config.rs index 292b065b4..e2cd167f5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -29,15 +29,6 @@ pub struct HttpTrackerConfig { pub ssl_key_path: Option, } -impl HttpTrackerConfig { - pub fn verify_ssl_cert_and_key_set(&self) -> bool { - self.ssl_cert_path.is_some() - && self.ssl_key_path.is_some() - && !self.ssl_cert_path.as_ref().unwrap().is_empty() - && !self.ssl_key_path.as_ref().unwrap().is_empty() - } -} - #[derive(Serialize, Deserialize)] pub struct HttpApiConfig { pub enabled: bool, @@ -48,20 +39,18 @@ pub struct HttpApiConfig { #[derive(Serialize, Deserialize)] pub struct Configuration { pub log_level: Option, - pub log_interval: Option, pub mode: TrackerMode, pub db_driver: DatabaseDrivers, pub db_path: String, - pub statistics: bool, + pub announce_interval: u32, + pub min_announce_interval: u32, + pub max_peer_timeout: u32, + pub on_reverse_proxy: bool, + pub external_ip: Option, + pub tracker_usage_statistics: bool, pub persistent_torrent_completed_stat: bool, - pub persistence_interval: u64, pub inactive_peer_cleanup_interval: u64, pub remove_peerless_torrents: bool, - pub external_ip: Option, - pub announce_interval: u32, - pub announce_interval_min: u32, - pub peer_timeout: u32, - pub on_reverse_proxy: bool, pub udp_trackers: Vec, pub http_trackers: Vec, pub http_api: HttpApiConfig, @@ -134,20 +123,18 @@ impl Configuration { pub fn default() -> Configuration { let mut configuration = Configuration { log_level: Option::from(String::from("info")), - log_interval: Some(60), mode: TrackerMode::Public, db_driver: DatabaseDrivers::Sqlite3, db_path: String::from("data.db"), - statistics: true, + announce_interval: 120, + min_announce_interval: 120, + max_peer_timeout: 900, + on_reverse_proxy: false, + external_ip: Some(String::from("0.0.0.0")), + tracker_usage_statistics: true, persistent_torrent_completed_stat: false, - persistence_interval: 0, inactive_peer_cleanup_interval: 600, remove_peerless_torrents: true, - external_ip: Some(String::from("0.0.0.0")), - announce_interval: 120, - announce_interval_min: 120, - peer_timeout: 900, - on_reverse_proxy: false, udp_trackers: Vec::new(), http_trackers: Vec::new(), http_api: HttpApiConfig { diff --git a/src/databases/database.rs b/src/databases/database.rs index e2f3cdfe5..721dfd00d 100644 --- a/src/databases/database.rs +++ b/src/databases/database.rs @@ -1,15 +1,11 @@ -use std::collections::BTreeMap; - use async_trait::async_trait; use derive_more::{Display, Error}; -use log::debug; use serde::{Deserialize, Serialize}; use crate::InfoHash; use crate::tracker::key::AuthKey; use crate::databases::mysql::MysqlDatabase; use crate::databases::sqlite::SqliteDatabase; -use crate::tracker::torrent::TorrentEntry; #[derive(Serialize, Deserialize, Debug)] pub enum DatabaseDrivers { @@ -18,8 +14,6 @@ pub enum DatabaseDrivers { } pub fn connect_database(db_driver: &DatabaseDrivers, db_path: &str) -> Result, r2d2::Error> { - debug!("{:?}", db_driver); - let database: Box = match db_driver { DatabaseDrivers::Sqlite3 => { let db = SqliteDatabase::new(db_path)?; @@ -44,7 +38,9 @@ pub trait Database: Sync + Send { async fn load_keys(&self) -> Result, Error>; - async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), Error>; + async fn load_whitelist(&self) -> Result, Error>; + + async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error>; async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result; diff --git a/src/databases/mysql.rs b/src/databases/mysql.rs index 6fd63b7d9..74f807d70 100644 --- a/src/databases/mysql.rs +++ b/src/databases/mysql.rs @@ -1,10 +1,9 @@ -use std::collections::BTreeMap; use std::str::FromStr; use async_trait::async_trait; use log::{debug}; use r2d2::Pool; -use r2d2_mysql::mysql::{Opts, OptsBuilder, params, TxOpts}; +use r2d2_mysql::mysql::{Opts, OptsBuilder, params}; use r2d2_mysql::mysql::prelude::Queryable; use r2d2_mysql::MysqlConnectionManager; @@ -12,7 +11,6 @@ use crate::{AUTH_KEY_LENGTH, InfoHash}; use crate::databases::database::{Database, Error}; use crate::databases::database; use crate::tracker::key::AuthKey; -use crate::tracker::torrent::TorrentEntry; pub struct MysqlDatabase { pool: Pool, @@ -37,20 +35,20 @@ impl Database for MysqlDatabase { let create_whitelist_table = " CREATE TABLE IF NOT EXISTS whitelist ( id integer PRIMARY KEY AUTO_INCREMENT, - info_hash BINARY(20) NOT NULL UNIQUE + info_hash VARCHAR(40) NOT NULL UNIQUE );".to_string(); let create_torrents_table = " CREATE TABLE IF NOT EXISTS torrents ( id integer PRIMARY KEY AUTO_INCREMENT, - info_hash BINARY(20) NOT NULL UNIQUE, + info_hash VARCHAR(40) NOT NULL UNIQUE, completed INTEGER DEFAULT 0 NOT NULL );".to_string(); let create_keys_table = format!(" CREATE TABLE IF NOT EXISTS `keys` ( `id` INT NOT NULL AUTO_INCREMENT, - `key` BINARY({}) NOT NULL, + `key` VARCHAR({}) NOT NULL, `valid_until` INT(10) NOT NULL, PRIMARY KEY (`id`), UNIQUE (`key`) @@ -68,7 +66,7 @@ impl Database for MysqlDatabase { async fn load_persistent_torrents(&self) -> Result, database::Error> { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT HEX(info_hash), completed FROM torrents", |(info_hash_string, completed): (String, u32)| { + let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT info_hash, completed FROM torrents", |(info_hash_string, completed): (String, u32)| { let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); (info_hash, completed) }).map_err(|_| database::Error::QueryReturnedNoRows)?; @@ -89,44 +87,38 @@ impl Database for MysqlDatabase { Ok(keys) } - async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), database::Error> { + async fn load_whitelist(&self) -> Result, Error> { + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; + + let info_hashes: Vec = conn.query_map("SELECT info_hash FROM whitelist", |info_hash: String| { + InfoHash::from_str(&info_hash).unwrap() + }).map_err(|_| database::Error::QueryReturnedNoRows)?; + + Ok(info_hashes) + } + + async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), database::Error> { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; + let info_hash_str = info_hash.to_string(); - let mut insert_vector= vec![]; + debug!("{}", info_hash_str); - for (info_hash, torrent_entry) in torrents { - let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - insert_vector.push(format!("(UNHEX('{}'), {})", info_hash.to_string(), completed.to_string())); - if insert_vector.len() == 1000 { - let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); - if db_transaction.query_drop(query).is_err() { - return Err(Error::InvalidQuery); - } - insert_vector.clear(); + match conn.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (:info_hash_str, :completed) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", params! { info_hash_str, completed }) { + Ok(_) => { + Ok(()) } - } - - if insert_vector.len() != 0 { - let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); - if db_transaction.query_drop(query).is_err() { - return Err(Error::InvalidQuery); + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) } - insert_vector.clear(); } - - if db_transaction.commit().is_err() { - return Err(Error::DatabaseError); - }; - - Ok(()) } async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - match conn.exec_first::("SELECT HEX(info_hash) FROM whitelist WHERE info_hash = UNHEX(:info_hash)", params! { info_hash => info_hash }) + match conn.exec_first::("SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", params! { info_hash }) .map_err(|_| database::Error::QueryReturnedNoRows)? { Some(info_hash) => { Ok(InfoHash::from_str(&info_hash).unwrap()) @@ -142,7 +134,7 @@ impl Database for MysqlDatabase { let info_hash_str = info_hash.to_string(); - match conn.exec_drop("INSERT INTO whitelist (info_hash) VALUES (UNHEX(:info_hash_str))", params! { info_hash_str }) { + match conn.exec_drop("INSERT INTO whitelist (info_hash) VALUES (:info_hash_str)", params! { info_hash_str }) { Ok(_) => { Ok(1) } @@ -158,7 +150,7 @@ impl Database for MysqlDatabase { let info_hash = info_hash.to_string(); - match conn.exec_drop("DELETE FROM whitelist WHERE info_hash = UNHEX(:info_hash)", params! { info_hash }) { + match conn.exec_drop("DELETE FROM whitelist WHERE info_hash = :info_hash", params! { info_hash }) { Ok(_) => { Ok(1) } diff --git a/src/databases/sqlite.rs b/src/databases/sqlite.rs index 9c452eadc..55c77969a 100644 --- a/src/databases/sqlite.rs +++ b/src/databases/sqlite.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeMap; use std::str::FromStr; use async_trait::async_trait; @@ -11,7 +10,6 @@ use crate::{AUTH_KEY_LENGTH, InfoHash}; use crate::databases::database::{Database, Error}; use crate::databases::database; use crate::tracker::key::AuthKey; -use crate::tracker::torrent::TorrentEntry; pub struct SqliteDatabase { pool: Pool, @@ -96,20 +94,35 @@ impl Database for SqliteDatabase { Ok(keys) } - async fn save_persistent_torrent_data(&self, torrents: &BTreeMap) -> Result<(), database::Error> { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + async fn load_whitelist(&self) -> Result, Error> { + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; - let db_transaction = conn.transaction()?; + let mut stmt = conn.prepare("SELECT info_hash FROM whitelist")?; - for (info_hash, torrent_entry) in torrents { - let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR IGNORE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); - let _ = db_transaction.execute("UPDATE torrents SET completed = ? WHERE info_hash = ?", &[completed.to_string(), info_hash.to_string()]); - } + let info_hash_iter = stmt.query_map(NO_PARAMS, |row| { + let info_hash: String = row.get(0)?; + + Ok(InfoHash::from_str(&info_hash).unwrap()) + })?; - let _ = db_transaction.commit(); + let info_hashes: Vec = info_hash_iter.filter_map(|x| x.ok()).collect(); + + Ok(info_hashes) + } + + async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), database::Error> { + let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; - Ok(()) + match conn.execute("INSERT INTO torrents (info_hash, completed) VALUES (?1, ?2) ON CONFLICT(info_hash) DO UPDATE SET completed = ?2", &[info_hash.to_string(), completed.to_string()]) { + Ok(updated) => { + if updated > 0 { return Ok(()); } + Err(database::Error::QueryReturnedNoRows) + } + Err(e) => { + debug!("{:?}", e); + Err(database::Error::InvalidQuery) + } + } } async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { diff --git a/src/http/filters.rs b/src/http/filters.rs index d5a7881d8..a288f8d97 100644 --- a/src/http/filters.rs +++ b/src/http/filters.rs @@ -3,7 +3,6 @@ use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; use std::sync::Arc; -use log::debug; use warp::{Filter, reject, Rejection}; use crate::{InfoHash, MAX_SCRAPE_TORRENTS, PeerId}; @@ -75,11 +74,9 @@ async fn info_hashes(raw_query: String) -> WebResult> { for v in split_raw_query { if v.contains("info_hash") { let raw_info_hash = v.split("=").collect::>()[1]; - debug!("Raw info hash: {}", raw_info_hash); let info_hash_bytes = percent_encoding::percent_decode_str(raw_info_hash).collect::>(); let info_hash = InfoHash::from_str(&hex::encode(info_hash_bytes)); if let Ok(ih) = info_hash { - debug!("Parsed info hash: {}", ih.to_string()); info_hashes.push(ih); } } @@ -151,8 +148,7 @@ async fn peer_addr((on_reverse_proxy, remote_addr, x_forwarded_for): (bool, Opti // set client ip to last forwarded ip let x_forwarded_ip = *x_forwarded_ips.last().unwrap(); - IpAddr::from_str(x_forwarded_ip).or_else(|e| { - debug!("{}", e); + IpAddr::from_str(x_forwarded_ip).or_else(|_| { Err(reject::custom(ServerError::AddressNotFound)) }) } diff --git a/src/http/handlers.rs b/src/http/handlers.rs index 977aeb4ba..7616ca301 100644 --- a/src/http/handlers.rs +++ b/src/http/handlers.rs @@ -56,7 +56,7 @@ pub async fn handle_announce(announce_request: AnnounceRequest, auth_key: Option IpAddr::V6(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Tcp6Announce).await; } } - send_announce_response(&announce_request, torrent_stats, peers, announce_interval, tracker.config.announce_interval_min) + send_announce_response(&announce_request, torrent_stats, peers, announce_interval, tracker.config.min_announce_interval) } /// Handle scrape request diff --git a/src/jobs/log_statistics.rs b/src/jobs/log_statistics.rs deleted file mode 100644 index f62399a47..000000000 --- a/src/jobs/log_statistics.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::sync::Arc; -use log::info; -use tokio::task::JoinHandle; -use crate::{Configuration}; -use crate::tracker::tracker::TorrentTracker; - -pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { - let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.log_interval.unwrap_or(60); - - tokio::spawn(async move { - let interval = std::time::Duration::from_secs(interval); - let mut interval = tokio::time::interval(interval); - interval.tick().await; - - loop { - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Stopping statistics logging job.."); - break; - } - _ = interval.tick() => { - if let Some(tracker) = weak_tracker.upgrade() { - tracker.post_log().await; - } else { - break; - } - } - } - } - }) -} diff --git a/src/jobs/mod.rs b/src/jobs/mod.rs index a71fcb210..c3e58e56e 100644 --- a/src/jobs/mod.rs +++ b/src/jobs/mod.rs @@ -1,6 +1,4 @@ -pub mod persistent_torrent_statistics; pub mod torrent_cleanup; pub mod tracker_api; -pub mod log_statistics; pub mod http_tracker; pub mod udp_tracker; diff --git a/src/jobs/persistent_torrent_statistics.rs b/src/jobs/persistent_torrent_statistics.rs deleted file mode 100644 index 54ee23b6b..000000000 --- a/src/jobs/persistent_torrent_statistics.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::sync::Arc; -use log::info; -use tokio::task::JoinHandle; -use crate::{Configuration}; -use crate::tracker::tracker::TorrentTracker; - -pub fn start_job(config: &Configuration, tracker: Arc) -> JoinHandle<()> { - let weak_tracker = std::sync::Arc::downgrade(&tracker); - let interval = config.persistence_interval; - - tokio::spawn(async move { - let interval = std::time::Duration::from_secs(interval); - let mut interval = tokio::time::interval(interval); - interval.tick().await; - - // periodically save torrents to database - loop { - tokio::select! { - _ = tokio::signal::ctrl_c() => { - // Save before shutting down - tracker.periodic_saving().await; - info!("Stopping periodic torrent saving job.."); - break; - } - _ = interval.tick() => { - if let Some(tracker) = weak_tracker.upgrade() { - info!("Saving torrents to database..."); - tracker.periodic_saving().await; - info!("Periodic saving done."); - } else { - // If tracker no longer exists, stop job - break; - } - } - } - } - }) -} diff --git a/src/protocol/utils.rs b/src/protocol/utils.rs index 2b8ac3ebf..30b87b99b 100644 --- a/src/protocol/utils.rs +++ b/src/protocol/utils.rs @@ -1,5 +1,3 @@ -use std::error::Error; -use std::fmt::Write; use std::net::SocketAddr; use std::time::SystemTime; diff --git a/src/setup.rs b/src/setup.rs index 69f2db432..ed9b6d8ff 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -1,24 +1,21 @@ use std::sync::Arc; -use log::{info, warn}; +use log::{warn}; use tokio::task::JoinHandle; use crate::{Configuration}; -use crate::jobs::{http_tracker, log_statistics, persistent_torrent_statistics, torrent_cleanup, tracker_api, udp_tracker}; +use crate::jobs::{http_tracker, torrent_cleanup, tracker_api, udp_tracker}; use crate::tracker::tracker::TorrentTracker; pub async fn setup(config: &Configuration, tracker: Arc) -> Vec>{ let mut jobs: Vec> = Vec::new(); + // Load peer keys if tracker.is_private() { - tracker.load_keys().await.expect("Could not retrieve keys."); + tracker.load_keys().await.expect("Could not retrieve keys from database."); } - // todo: replace by realtime updates - // Load persistent torrents - if config.persistent_torrent_completed_stat && config.persistence_interval > 0 { - info!("Loading persistent torrents into memory.."); - tracker.load_persistent_torrents().await.expect("Could not load persistent torrents."); - info!("Persistent torrents loaded."); - jobs.push(persistent_torrent_statistics::start_job(&config, tracker.clone())); + // Load whitelisted torrents + if tracker.is_whitelisted() { + tracker.load_whitelist().await.expect("Could not load whitelist from database."); } // Start the UDP blocks @@ -48,12 +45,5 @@ pub async fn setup(config: &Configuration, tracker: Arc) -> Vec< jobs.push(torrent_cleanup::start_job(&config, tracker.clone())); } - // Log detailed torrent stats - if let Some(log_interval) = config.log_interval { - if log_interval > 0 { - jobs.push(log_statistics::start_job(&config, tracker.clone())); - } - } - jobs } diff --git a/src/tracker/statistics.rs b/src/tracker/statistics.rs index 5822d2d4e..c67df72ec 100644 --- a/src/tracker/statistics.rs +++ b/src/tracker/statistics.rs @@ -109,12 +109,24 @@ impl StatsTracker { stats_lock.tcp6_scrapes_handled += 1; stats_lock.tcp6_connections_handled += 1; } - TrackerStatisticsEvent::Udp4Connect => { stats_lock.udp4_connections_handled += 1; } - TrackerStatisticsEvent::Udp4Announce => { stats_lock.udp4_announces_handled += 1; } - TrackerStatisticsEvent::Udp4Scrape => { stats_lock.udp4_scrapes_handled += 1; } - TrackerStatisticsEvent::Udp6Connect => { stats_lock.udp6_connections_handled += 1; } - TrackerStatisticsEvent::Udp6Announce => { stats_lock.udp6_announces_handled += 1; } - TrackerStatisticsEvent::Udp6Scrape => { stats_lock.udp6_scrapes_handled += 1; } + TrackerStatisticsEvent::Udp4Connect => { + stats_lock.udp4_connections_handled += 1; + } + TrackerStatisticsEvent::Udp4Announce => { + stats_lock.udp4_announces_handled += 1; + } + TrackerStatisticsEvent::Udp4Scrape => { + stats_lock.udp4_scrapes_handled += 1; + } + TrackerStatisticsEvent::Udp6Connect => { + stats_lock.udp6_connections_handled += 1; + } + TrackerStatisticsEvent::Udp6Announce => { + stats_lock.udp6_announces_handled += 1; + } + TrackerStatisticsEvent::Udp6Scrape => { + stats_lock.udp6_scrapes_handled += 1; + } } drop(stats_lock); diff --git a/src/tracker/torrent.rs b/src/tracker/torrent.rs index 4b891e992..0c03e3f82 100644 --- a/src/tracker/torrent.rs +++ b/src/tracker/torrent.rs @@ -21,7 +21,10 @@ impl TorrentEntry { } } - pub fn update_peer(&mut self, peer: &TorrentPeer) { + // Update peer and return completed (times torrent has been downloaded) + pub fn update_peer(&mut self, peer: &TorrentPeer) -> bool { + let mut did_torrent_stats_change: bool = false; + match peer.event { AnnounceEvent::Stopped => { let _ = self.peers.remove(&peer.peer_id); @@ -29,12 +32,17 @@ impl TorrentEntry { AnnounceEvent::Completed => { let peer_old = self.peers.insert(peer.peer_id.clone(), peer.clone()); // Don't count if peer was not previously known - if peer_old.is_some() { self.completed += 1; } + if peer_old.is_some() { + self.completed += 1; + did_torrent_stats_change = true; + } } _ => { let _ = self.peers.insert(peer.peer_id.clone(), peer.clone()); } } + + did_torrent_stats_change } pub fn get_peers(&self, client_addr: Option<&SocketAddr>) -> Vec<&TorrentPeer> { diff --git a/src/tracker/tracker.rs b/src/tracker/tracker.rs index 45b23f728..bcfae3c37 100644 --- a/src/tracker/tracker.rs +++ b/src/tracker/tracker.rs @@ -3,7 +3,6 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use log::info; use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::sync::mpsc::error::SendError; @@ -22,9 +21,8 @@ pub struct TorrentTracker { pub config: Arc, mode: TrackerMode, keys: RwLock>, + whitelist: RwLock>, torrents: RwLock>, - updates: RwLock>, - shadow: RwLock>, stats_tracker: StatsTracker, database: Box } @@ -35,15 +33,14 @@ impl TorrentTracker { let mut stats_tracker = StatsTracker::new(); // starts a thread for updating tracker stats - if config.statistics { stats_tracker.run_worker(); } + if config.tracker_usage_statistics { stats_tracker.run_worker(); } Ok(TorrentTracker { config: config.clone(), mode: config.mode, keys: RwLock::new(std::collections::HashMap::new()), + whitelist: RwLock::new(std::collections::HashSet::new()), torrents: RwLock::new(std::collections::BTreeMap::new()), - updates: RwLock::new(std::collections::HashMap::new()), - shadow: RwLock::new(std::collections::HashMap::new()), stats_tracker, database }) @@ -64,7 +61,7 @@ impl TorrentTracker { pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { let auth_key = key::generate_auth_key(seconds_valid); - // add key to database + // Add key to database if let Err(error) = self.database.add_key_to_keys(&auth_key).await { return Err(error); } // Add key to in-memory database @@ -125,6 +122,8 @@ impl TorrentTracker { let keys_from_database = self.database.load_keys().await?; let mut keys = self.keys.write().await; + keys.clear(); + for key in keys_from_database { let _ = keys.insert(key.key.clone(), key); } @@ -132,6 +131,19 @@ impl TorrentTracker { Ok(()) } + pub async fn load_whitelist(&self) -> Result<(), database::Error> { + let whitelisted_torrents_from_database = self.database.load_whitelist().await?; + let mut whitelist = self.whitelist.write().await; + + whitelist.clear(); + + for info_hash in whitelisted_torrents_from_database { + let _ = whitelist.insert(info_hash); + } + + Ok(()) + } + // Loading the torrents from database into memory pub async fn load_persistent_torrents(&self) -> Result<(), database::Error> { let persistent_torrents = self.database.load_persistent_torrents().await?; @@ -152,12 +164,6 @@ impl TorrentTracker { Ok(()) } - // Saving the torrents from memory - pub async fn save_torrents(&self) -> Result<(), database::Error> { - let torrents = self.torrents.read().await; - self.database.save_persistent_torrent_data(&*torrents).await - } - // Adding torrents is not relevant to public trackers. pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result { self.database.add_info_hash_to_whitelist(info_hash.clone()).await @@ -199,19 +205,15 @@ impl TorrentTracker { } }; - torrent_entry.update_peer(peer); + let stats_updated = torrent_entry.update_peer(peer); - let (seeders, completed, leechers) = torrent_entry.get_stats(); - - if self.config.persistent_torrent_completed_stat { - let mut updates = self.updates.write().await; - if updates.contains_key(info_hash) { - updates.remove(info_hash); - } - updates.insert(*info_hash, completed); - drop(updates); + // todo: move this action to a separate worker + if self.config.persistent_torrent_completed_stat && stats_updated { + let _ = self.database.save_persistent_torrent(&info_hash, torrent_entry.completed).await; } + let (seeders, completed, leechers) = torrent_entry.get_stats(); + TorrentStats { seeders, leechers, @@ -231,19 +233,6 @@ impl TorrentTracker { self.stats_tracker.send_event(event).await } - pub async fn post_log(&self) { - let torrents = self.torrents.read().await; - let torrents_size = torrents.len(); - drop(torrents); - let updates = self.updates.read().await; - let updates_size = updates.len(); - drop(updates); - let shadow = self.shadow.read().await; - let shadow_size = shadow.len(); - drop(shadow); - info!("-=[ Stats ]=- | Torrents: {} | Updates: {} | Shadow: {}", torrents_size, updates_size, shadow_size); - } - // Remove inactive peers and (optionally) peerless torrents pub async fn cleanup_torrents(&self) { let mut torrents_lock = self.torrents.write().await; @@ -251,7 +240,7 @@ impl TorrentTracker { // If we don't need to remove torrents we will use the faster iter if self.config.remove_peerless_torrents { torrents_lock.retain(|_, torrent_entry| { - torrent_entry.remove_inactive_peers(self.config.peer_timeout); + torrent_entry.remove_inactive_peers(self.config.max_peer_timeout); match self.config.persistent_torrent_completed_stat { true => { torrent_entry.completed > 0 || torrent_entry.peers.len() > 0 } @@ -260,61 +249,8 @@ impl TorrentTracker { }); } else { for (_, torrent_entry) in torrents_lock.iter_mut() { - torrent_entry.remove_inactive_peers(self.config.peer_timeout); + torrent_entry.remove_inactive_peers(self.config.max_peer_timeout); } } } - - // todo: refactor - pub async fn periodic_saving(&self) { - // Get a lock for writing - // let mut shadow = self.shadow.write().await; - - // We will get the data and insert it into the shadow, while clearing updates. - let mut updates = self.updates.write().await; - let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); - // let mut torrent_hashes: Vec = Vec::new(); - // Copying updates to updates_cloned - for (k, completed) in updates.iter() { - updates_cloned.insert(k.clone(), completed.clone()); - } - updates.clear(); - drop(updates); - - // Copying updates_cloned into the shadow to overwrite - for (k, completed) in updates_cloned.iter() { - let mut shadows = self.shadow.write().await; - if shadows.contains_key(k) { - shadows.remove(k); - } - shadows.insert(k.clone(), completed.clone()); - drop(shadows); - } - drop(updates_cloned); - - // We updated the shadow data from the updates data, let's handle shadow data as expected. - // Handle shadow_copy to be updated into SQL - let mut shadow_copy: BTreeMap = BTreeMap::new(); - let shadows = self.shadow.read().await; - for (infohash, completed) in shadows.iter() { - shadow_copy.insert(infohash.clone(), TorrentEntry { - peers: Default::default(), - completed: completed.clone(), - }); - } - drop(shadows); - - // We will now save the data from the shadow into the database. - // This should not put any strain on the server itself, other then the harddisk/ssd. - info!("Start saving shadow data into SQL..."); - let result = self.database.save_persistent_torrent_data(&shadow_copy).await; - if result.is_ok() { - info!("Done saving data to SQL and succeeded, emptying shadow..."); - let mut shadow = self.shadow.write().await; - shadow.clear(); - drop(shadow); - } else { - info!("Done saving data to SQL and failed, not emptying shadow..."); - } - } } From 0a771314295c79bc5c89bdbcdb35057df085d127 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 20:05:11 +0200 Subject: [PATCH 48/53] fix: fixed up some database errors --- src/databases/mysql.rs | 18 +++++++++--------- src/databases/sqlite.rs | 38 +++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/databases/mysql.rs b/src/databases/mysql.rs index 74f807d70..5b6e34eb1 100644 --- a/src/databases/mysql.rs +++ b/src/databases/mysql.rs @@ -54,7 +54,7 @@ impl Database for MysqlDatabase { UNIQUE (`key`) );", AUTH_KEY_LENGTH as i8); - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; conn.query_drop(&create_torrents_table).expect("Could not create torrents table."); conn.query_drop(&create_keys_table).expect("Could not create keys table."); @@ -64,7 +64,7 @@ impl Database for MysqlDatabase { } async fn load_persistent_torrents(&self) -> Result, database::Error> { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let torrents: Vec<(InfoHash, u32)> = conn.query_map("SELECT info_hash, completed FROM torrents", |(info_hash_string, completed): (String, u32)| { let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); @@ -98,7 +98,7 @@ impl Database for MysqlDatabase { } async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), database::Error> { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let info_hash_str = info_hash.to_string(); @@ -116,7 +116,7 @@ impl Database for MysqlDatabase { } async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.exec_first::("SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", params! { info_hash }) .map_err(|_| database::Error::QueryReturnedNoRows)? { @@ -130,7 +130,7 @@ impl Database for MysqlDatabase { } async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let info_hash_str = info_hash.to_string(); @@ -146,7 +146,7 @@ impl Database for MysqlDatabase { } async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let info_hash = info_hash.to_string(); @@ -162,7 +162,7 @@ impl Database for MysqlDatabase { } async fn get_key_from_keys(&self, key: &str) -> Result { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.exec_first::<(String, i64), _, _>("SELECT `key`, valid_until FROM `keys` WHERE `key` = :key", params! { key }) .map_err(|_| database::Error::QueryReturnedNoRows)? { @@ -179,7 +179,7 @@ impl Database for MysqlDatabase { } async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let key = auth_key.key.to_string(); let valid_until = auth_key.valid_until.unwrap_or(0).to_string(); @@ -196,7 +196,7 @@ impl Database for MysqlDatabase { } async fn remove_key_from_keys(&self, key: &str) -> Result { - let mut conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let mut conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.exec_drop("DELETE FROM `keys` WHERE key = :key", params! { key }) { Ok(_) => { diff --git a/src/databases/sqlite.rs b/src/databases/sqlite.rs index 55c77969a..e1659d897 100644 --- a/src/databases/sqlite.rs +++ b/src/databases/sqlite.rs @@ -6,7 +6,7 @@ use r2d2::Pool; use r2d2_sqlite::SqliteConnectionManager; use r2d2_sqlite::rusqlite::NO_PARAMS; -use crate::{AUTH_KEY_LENGTH, InfoHash}; +use crate::{InfoHash}; use crate::databases::database::{Database, Error}; use crate::databases::database; use crate::tracker::key::AuthKey; @@ -30,25 +30,25 @@ impl Database for SqliteDatabase { fn create_database_tables(&self) -> Result<(), database::Error> { let create_whitelist_table = " CREATE TABLE IF NOT EXISTS whitelist ( - id integer PRIMARY KEY AUTOINCREMENT, - info_hash VARCHAR(20) NOT NULL UNIQUE + id INTEGER PRIMARY KEY AUTOINCREMENT, + info_hash TEXT NOT NULL UNIQUE );".to_string(); let create_torrents_table = " CREATE TABLE IF NOT EXISTS torrents ( - id integer PRIMARY KEY AUTOINCREMENT, - info_hash VARCHAR(20) NOT NULL UNIQUE, + id INTEGER PRIMARY KEY AUTOINCREMENT, + info_hash TEXT NOT NULL UNIQUE, completed INTEGER DEFAULT 0 NOT NULL );".to_string(); - let create_keys_table = format!(" + let create_keys_table = " CREATE TABLE IF NOT EXISTS keys ( - id integer PRIMARY KEY AUTOINCREMENT, - key VARCHAR({}) NOT NULL UNIQUE, - valid_until INT(10) NOT NULL - );", AUTH_KEY_LENGTH as i8); + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT NOT NULL UNIQUE, + valid_until INTEGER NOT NULL + );".to_string(); - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; conn.execute(&create_whitelist_table, NO_PARAMS) .and_then(|_| conn.execute(&create_keys_table, NO_PARAMS)) @@ -58,7 +58,7 @@ impl Database for SqliteDatabase { } async fn load_persistent_torrents(&self) -> Result, database::Error> { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let mut stmt = conn.prepare("SELECT info_hash, completed FROM torrents")?; @@ -111,7 +111,7 @@ impl Database for SqliteDatabase { } async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), database::Error> { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.execute("INSERT INTO torrents (info_hash, completed) VALUES (?1, ?2) ON CONFLICT(info_hash) DO UPDATE SET completed = ?2", &[info_hash.to_string(), completed.to_string()]) { Ok(updated) => { @@ -126,7 +126,7 @@ impl Database for SqliteDatabase { } async fn get_info_hash_from_whitelist(&self, info_hash: &str) -> Result { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let mut stmt = conn.prepare("SELECT info_hash FROM whitelist WHERE info_hash = ?")?; let mut rows = stmt.query(&[info_hash])?; @@ -142,7 +142,7 @@ impl Database for SqliteDatabase { } async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.execute("INSERT INTO whitelist (info_hash) VALUES (?)", &[info_hash.to_string()]) { Ok(updated) => { @@ -157,7 +157,7 @@ impl Database for SqliteDatabase { } async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.execute("DELETE FROM whitelist WHERE info_hash = ?", &[info_hash.to_string()]) { Ok(updated) => { @@ -172,7 +172,7 @@ impl Database for SqliteDatabase { } async fn get_key_from_keys(&self, key: &str) -> Result { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; let mut stmt = conn.prepare("SELECT key, valid_until FROM keys WHERE key = ?")?; let mut rows = stmt.query(&[key.to_string()])?; @@ -191,7 +191,7 @@ impl Database for SqliteDatabase { } async fn add_key_to_keys(&self, auth_key: &AuthKey) -> Result { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.execute("INSERT INTO keys (key, valid_until) VALUES (?1, ?2)", &[auth_key.key.to_string(), auth_key.valid_until.unwrap().to_string()], @@ -208,7 +208,7 @@ impl Database for SqliteDatabase { } async fn remove_key_from_keys(&self, key: &str) -> Result { - let conn = self.pool.get().map_err(|_| database::Error::InvalidQuery)?; + let conn = self.pool.get().map_err(|_| database::Error::DatabaseError)?; match conn.execute("DELETE FROM keys WHERE key = ?", &[key]) { Ok(updated) => { From 68d3242207d9a5d76114866a9acabe9cf0cae9e9 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 20:30:05 +0200 Subject: [PATCH 49/53] some code cleanup --- src/api/server.rs | 4 ++ src/tracker/tracker.rs | 110 ++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 61 deletions(-) diff --git a/src/api/server.rs b/src/api/server.rs index 77496b497..58c3f0d1a 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -152,7 +152,9 @@ pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp udp6_announces_handled: 0, udp6_scrapes_handled: 0, }; + let db = tracker.get_torrents().await; + let _: Vec<_> = db .iter() .map(|(_info_hash, torrent_entry)| { @@ -163,7 +165,9 @@ pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp results.torrents += 1; }) .collect(); + let stats = tracker.get_stats().await; + results.tcp4_connections_handled = stats.tcp4_connections_handled as u32; results.tcp4_announces_handled = stats.tcp4_announces_handled as u32; results.tcp4_scrapes_handled = stats.tcp4_scrapes_handled as u32; diff --git a/src/tracker/tracker.rs b/src/tracker/tracker.rs index bcfae3c37..163bfe446 100644 --- a/src/tracker/tracker.rs +++ b/src/tracker/tracker.rs @@ -60,33 +60,66 @@ impl TorrentTracker { pub async fn generate_auth_key(&self, seconds_valid: u64) -> Result { let auth_key = key::generate_auth_key(seconds_valid); - - // Add key to database - if let Err(error) = self.database.add_key_to_keys(&auth_key).await { return Err(error); } - - // Add key to in-memory database + self.database.add_key_to_keys(&auth_key).await?; self.keys.write().await.insert(auth_key.key.clone(), auth_key.clone()); - Ok(auth_key) } - pub async fn remove_auth_key(&self, key: &str) -> Result { + pub async fn remove_auth_key(&self, key: &str) -> Result<(), database::Error> { self.database.remove_key_from_keys(&key).await?; - - // Remove key from in-memory database self.keys.write().await.remove(key); - - Ok(1) + Ok(()) } pub async fn verify_auth_key(&self, auth_key: &AuthKey) -> Result<(), key::Error> { - let keys_lock = self.keys.read().await; + match self.keys.read().await.get(&auth_key.key) { + None => Err(key::Error::KeyInvalid), + Some(key) => key::verify_auth_key(key) + } + } - if let Some(key) = keys_lock.get(&auth_key.key) { - key::verify_auth_key(key) - } else { - Err(key::Error::KeyInvalid) + pub async fn load_keys(&self) -> Result<(), database::Error> { + let keys_from_database = self.database.load_keys().await?; + let mut keys = self.keys.write().await; + + keys.clear(); + + for key in keys_from_database { + let _ = keys.insert(key.key.clone(), key); + } + + Ok(()) + } + + // Adding torrents is not relevant to public trackers. + pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result<(), database::Error> { + self.database.add_info_hash_to_whitelist(info_hash.clone()).await?; + self.whitelist.write().await.insert(info_hash.clone()); + Ok(()) + } + + // Removing torrents is not relevant to public trackers. + pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result<(), database::Error> { + self.database.remove_info_hash_from_whitelist(info_hash.clone()).await?; + self.whitelist.write().await.remove(info_hash); + Ok(()) + } + + pub async fn is_info_hash_whitelisted(&self, info_hash: &InfoHash) -> bool { + self.whitelist.read().await.contains(info_hash) + } + + pub async fn load_whitelist(&self) -> Result<(), database::Error> { + let whitelisted_torrents_from_database = self.database.load_whitelist().await?; + let mut whitelist = self.whitelist.write().await; + + whitelist.clear(); + + for info_hash in whitelisted_torrents_from_database { + let _ = whitelist.insert(info_hash); } + + Ok(()) } pub async fn authenticate_request(&self, info_hash: &InfoHash, key: &Option) -> Result<(), TorrentError> { @@ -107,7 +140,6 @@ impl TorrentTracker { } } - // todo: speed this up // check if info_hash is whitelisted if self.is_whitelisted() { if !self.is_info_hash_whitelisted(info_hash).await { @@ -118,32 +150,6 @@ impl TorrentTracker { Ok(()) } - pub async fn load_keys(&self) -> Result<(), database::Error> { - let keys_from_database = self.database.load_keys().await?; - let mut keys = self.keys.write().await; - - keys.clear(); - - for key in keys_from_database { - let _ = keys.insert(key.key.clone(), key); - } - - Ok(()) - } - - pub async fn load_whitelist(&self) -> Result<(), database::Error> { - let whitelisted_torrents_from_database = self.database.load_whitelist().await?; - let mut whitelist = self.whitelist.write().await; - - whitelist.clear(); - - for info_hash in whitelisted_torrents_from_database { - let _ = whitelist.insert(info_hash); - } - - Ok(()) - } - // Loading the torrents from database into memory pub async fn load_persistent_torrents(&self) -> Result<(), database::Error> { let persistent_torrents = self.database.load_persistent_torrents().await?; @@ -164,24 +170,6 @@ impl TorrentTracker { Ok(()) } - // Adding torrents is not relevant to public trackers. - pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result { - self.database.add_info_hash_to_whitelist(info_hash.clone()).await - } - - // Removing torrents is not relevant to public trackers. - pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result { - self.database.remove_info_hash_from_whitelist(info_hash.clone()).await - } - - pub async fn is_info_hash_whitelisted(&self, info_hash: &InfoHash) -> bool { - match self.database.get_info_hash_from_whitelist(&info_hash.to_string()).await { - Ok(_) => true, - Err(_) => false - } - } - - pub async fn get_torrent_peers(&self, info_hash: &InfoHash, client_addr: &SocketAddr, ) -> Vec { let read_lock = self.torrents.read().await; From ce24446cbb1ed49acdbd9104245b1f5b70e027ab Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 21:41:54 +0200 Subject: [PATCH 50/53] feat: scrape unauthenticated info_hashes return 0 values instead of error --- src/http/handlers.rs | 11 ++++++----- src/udp/handlers.rs | 24 ++++++++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/http/handlers.rs b/src/http/handlers.rs index 7616ca301..0dc737641 100644 --- a/src/http/handlers.rs +++ b/src/http/handlers.rs @@ -65,13 +65,14 @@ pub async fn handle_scrape(scrape_request: ScrapeRequest, auth_key: Option { - let (seeders, completed, leechers) = torrent_info.get_stats(); - ScrapeResponseEntry { complete: seeders, downloaded: completed, incomplete: leechers } + if authenticate(info_hash, &auth_key, tracker.clone()).await.is_ok() { + let (seeders, completed, leechers) = torrent_info.get_stats(); + ScrapeResponseEntry { complete: seeders, downloaded: completed, incomplete: leechers } + } else { + ScrapeResponseEntry { complete: 0, downloaded: 0, incomplete: 0 } + } } None => { ScrapeResponseEntry { complete: 0, downloaded: 0, incomplete: 0 } diff --git a/src/udp/handlers.rs b/src/udp/handlers.rs index 4ea767c0b..860a2fe4b 100644 --- a/src/udp/handlers.rs +++ b/src/udp/handlers.rs @@ -154,16 +154,22 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra for info_hash in request.info_hashes.iter() { let info_hash = InfoHash(info_hash.0); - if authenticate(&info_hash, tracker.clone()).await.is_err() { continue; } - let scrape_entry = match db.get(&info_hash) { Some(torrent_info) => { - let (seeders, completed, leechers) = torrent_info.get_stats(); - - TorrentScrapeStatistics { - seeders: NumberOfPeers(seeders as i32), - completed: NumberOfDownloads(completed as i32), - leechers: NumberOfPeers(leechers as i32), + if authenticate(&info_hash, tracker.clone()).await.is_ok() { + let (seeders, completed, leechers) = torrent_info.get_stats(); + + TorrentScrapeStatistics { + seeders: NumberOfPeers(seeders as i32), + completed: NumberOfDownloads(completed as i32), + leechers: NumberOfPeers(leechers as i32), + } + } else { + TorrentScrapeStatistics { + seeders: NumberOfPeers(0), + completed: NumberOfDownloads(0), + leechers: NumberOfPeers(0), + } } } None => { @@ -178,6 +184,8 @@ pub async fn handle_scrape(remote_addr: SocketAddr, request: &ScrapeRequest, tra torrent_stats.push(scrape_entry); } + drop(db); + // send stats event match remote_addr { SocketAddr::V4(_) => { tracker.send_stats_event(TrackerStatisticsEvent::Udp4Scrape).await; } From 77e173cf2b7e0e075efff5b8619fc1c1ecf0a84c Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 21:48:19 +0200 Subject: [PATCH 51/53] refactor: removed root from http routes --- src/http/routes.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/http/routes.rs b/src/http/routes.rs index a0b197f44..53b2b0ce5 100644 --- a/src/http/routes.rs +++ b/src/http/routes.rs @@ -14,22 +14,11 @@ use crate::tracker::tracker::TorrentTracker; /// All routes pub fn routes(tracker: Arc) -> impl Filter + Clone { - root(tracker.clone()) - .or(announce(tracker.clone())) - .or(scrape(tracker.clone())) + announce(tracker.clone()) + .or(scrape(tracker)) .recover(send_error) } -/// GET / or / -fn root(tracker: Arc) -> impl Filter + Clone { - warp::any() - .and(warp::filters::method::get()) - .and(with_announce_request(tracker.config.on_reverse_proxy)) - .and(with_auth_key()) - .and(with_tracker(tracker)) - .and_then(handle_announce) -} - /// GET /announce or /announce/ fn announce(tracker: Arc) -> impl Filter + Clone { warp::path::path("announce") From b9aca67246df50f6ed183f7eb61652c54dc3a40c Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 22:02:54 +0200 Subject: [PATCH 52/53] feat: added endpoints to reload whitelist and keys from database --- src/api/server.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/api/server.rs b/src/api/server.rs index 58c3f0d1a..19ceac92a 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -300,6 +300,46 @@ pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp } }); + // GET /api/whitelist/reload + // Reload whitelist + let t7 = tracker.clone(); + let reload_whitelist = filters::method::get() + .and(filters::path::path("whitelist")) + .and(filters::path::path("reload")) + .and(filters::path::end()) + .map(move || { + let tracker = t7.clone(); + tracker + }) + .and_then(|tracker: Arc| { + async move { + match tracker.load_whitelist().await { + Ok(_) => Ok(warp::reply::json(&ActionStatus::Ok)), + Err(_) => Err(warp::reject::custom(ActionStatus::Err { reason: "failed to reload whitelist".into() })) + } + } + }); + + // GET /api/keys/reload + // Reload whitelist + let t8 = tracker.clone(); + let reload_keys = filters::method::get() + .and(filters::path::path("keys")) + .and(filters::path::path("reload")) + .and(filters::path::end()) + .map(move || { + let tracker = t8.clone(); + tracker + }) + .and_then(|tracker: Arc| { + async move { + match tracker.load_keys().await { + Ok(_) => Ok(warp::reply::json(&ActionStatus::Ok)), + Err(_) => Err(warp::reject::custom(ActionStatus::Err { reason: "failed to reload keys".into() })) + } + } + }); + let api_routes = filters::path::path("api") .and(view_torrent_list @@ -309,6 +349,8 @@ pub fn start(socket_addr: SocketAddr, tracker: Arc) -> impl warp .or(add_torrent) .or(create_key) .or(delete_key) + .or(reload_whitelist) + .or(reload_keys) ); let server = api_routes.and(authenticate(tracker.config.http_api.access_tokens.clone())); From df87a9d3e89e530bfe90f1a48d0df884d377b430 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 9 May 2022 22:04:28 +0200 Subject: [PATCH 53/53] updated example config --- README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index bb4649271..beb2591ea 100644 --- a/README.md +++ b/README.md @@ -52,29 +52,26 @@ cargo build --release ```toml log_level = "info" mode = "public" +db_driver = "Sqlite3" db_path = "data.db" -persistence = false -cleanup_interval = 600 -cleanup_peerless = true -external_ip = "0.0.0.0" announce_interval = 120 -announce_interval_min = 900 -peer_timeout = 900 +min_announce_interval = 120 +max_peer_timeout = 900 on_reverse_proxy = false +external_ip = "0.0.0.0" +tracker_usage_statistics = true +persistent_torrent_completed_stat = false +inactive_peer_cleanup_interval = 600 +remove_peerless_torrents = true [[udp_trackers]] enabled = false bind_address = "0.0.0.0:6969" -[[udp_trackers]] -enabled = true -bind_address = "[::]:6969" - [[http_trackers]] enabled = true bind_address = "0.0.0.0:6969" ssl_enabled = false -ssl_bind_address = "0.0.0.0:6868" ssl_cert_path = "" ssl_key_path = ""