diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index ada96f77f..9a4f90990 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -16,6 +16,8 @@ jobs: env: CARGO_INCREMENTAL: "0" RUSTFLAGS: "-Cinstrument-coverage" + TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST: "true" + TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST: "true" steps: - name: Checkout repository diff --git a/.github/workflows/generate_coverage_pr.yaml b/.github/workflows/generate_coverage_pr.yaml index e07a5a755..b25e785bb 100644 --- a/.github/workflows/generate_coverage_pr.yaml +++ b/.github/workflows/generate_coverage_pr.yaml @@ -16,6 +16,8 @@ jobs: env: CARGO_INCREMENTAL: "0" RUSTFLAGS: "-Cinstrument-coverage" + TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST: "true" + TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST: "true" steps: - name: Checkout repository diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 83a290663..c69307242 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -137,6 +137,10 @@ jobs: name: Run MySQL Database Tests run: TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST=true cargo test --package bittorrent-tracker-core + - id: database-postgresql + name: Run PostgreSQL Database Tests + run: TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST=true cargo test --package bittorrent-tracker-core + e2e: name: E2E runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index fd83ee918..044190093 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .env +__pycache__/ +*.pyc *.code-workspace **/*.rs.bk /.coverage/ diff --git a/Cargo.lock b/Cargo.lock index 03138f718..c0215fb4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,17 +17,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "version_check", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -182,12 +171,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - [[package]] name = "astral-tokio-tar" version = "0.6.0" @@ -374,6 +357,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.6.1" @@ -576,17 +568,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "bigdecimal" -version = "0.4.10" +name = "base64ct" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" -dependencies = [ - "autocfg", - "libm", - "num-bigint", - "num-integer", - "num-traits", -] +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "binascii" @@ -594,24 +579,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" -[[package]] -name = "bindgen" -version = "0.72.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.117", -] - [[package]] name = "bit-vec" version = "0.4.4" @@ -623,6 +590,9 @@ name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +dependencies = [ + "serde_core", +] [[package]] name = "bittorrent-http-tracker-core" @@ -713,17 +683,16 @@ name = "bittorrent-tracker-core" version = "3.0.0-develop" dependencies = [ "aquatic_udp_protocol", + "async-trait", "bittorrent-primitives", "chrono", "derive_more", "local-ip-address", "mockall", - "r2d2", - "r2d2_mysql", - "r2d2_sqlite", "rand 0.10.1", "serde", "serde_json", + "sqlx", "testcontainers", "thiserror 2.0.18", "tokio", @@ -781,18 +750,6 @@ dependencies = [ "torrust-tracker-primitives", ] -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -908,30 +865,6 @@ dependencies = [ "time", ] -[[package]] -name = "borsh" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" -dependencies = [ - "borsh-derive", - "bytes", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "brotli" version = "8.0.2" @@ -953,49 +886,12 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "btoi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" -dependencies = [ - "num-traits", -] - -[[package]] -name = "bufstream" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" - [[package]] name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytemuck" version = "1.25.0" @@ -1056,15 +952,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.4" @@ -1137,17 +1024,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.6.1" @@ -1255,6 +1131,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "convert_case" version = "0.10.0" @@ -1308,6 +1190,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1391,28 +1288,6 @@ dependencies = [ "itertools 0.13.0", ] -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1565,6 +1440,17 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1629,17 +1515,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "derive_utils" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "362f47930db19fe7735f527e6595e4900316b893ebf6d48ad3d31be928d57dd6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "diff" version = "0.1.13" @@ -1653,7 +1528,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1678,6 +1555,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast" version = "0.11.0" @@ -1701,6 +1584,9 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -1747,6 +1633,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "etcetera" version = "0.11.0" @@ -1784,18 +1681,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "2.4.1" @@ -1853,10 +1738,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "libz-sys", "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1869,27 +1764,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1928,62 +1802,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "frunk" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28aef0f9aa070bce60767c12ba9cb41efeaf1a2bc6427f87b7d83f11239a16d7" -dependencies = [ - "frunk_core", - "frunk_derives", - "frunk_proc_macros", - "serde", -] - -[[package]] -name = "frunk_core" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476eeaa382e3462b84da5d6ba3da97b5786823c2d0d3a0d04ef088d073da225c" -dependencies = [ - "serde", -] - -[[package]] -name = "frunk_derives" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b4095fc99e1d858e5b8c7125d2638372ec85aa0fe6c807105cf10b0265ca6c" -dependencies = [ - "frunk_proc_macro_helpers", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "frunk_proc_macro_helpers" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1952b802269f2db12ab7c0bd328d0ae8feaabf19f352a7b0af7bb0c5693abfce" -dependencies = [ - "frunk_core", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "frunk_proc_macros" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3462f590fa236005bd7ca4847f81438bd6fe0febd4d04e11968d4c2e96437e78" -dependencies = [ - "frunk_core", - "frunk_proc_macro_helpers", - "quote", - "syn 2.0.117", -] - [[package]] name = "fs-err" version = "3.3.0" @@ -2000,12 +1818,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.32" @@ -2048,6 +1860,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -2235,9 +2058,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash", -] [[package]] name = "hashbrown" @@ -2253,16 +2073,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.1.5", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -2273,11 +2084,11 @@ checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "hashlink" -version = "0.11.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.16.1", + "hashbrown 0.15.5", ] [[package]] @@ -2304,6 +2115,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.12" @@ -2449,7 +2278,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "system-configuration", "tokio", "tower-service", @@ -2649,15 +2478,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "io-enum" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de9008599afe8527a8c9d70423437363b321649161e98473f433de802d76107" -dependencies = [ - "derive_utils", -] - [[package]] name = "ipnet" version = "2.12.0" @@ -2810,6 +2630,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "leb128fmt" @@ -2823,16 +2646,6 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" -[[package]] -name = "libloading" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" -dependencies = [ - "cfg-if", - "windows-link", -] - [[package]] name = "libm" version = "0.2.16" @@ -2853,20 +2666,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.37.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -2914,15 +2716,6 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -2935,6 +2728,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" @@ -2977,12 +2780,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3039,114 +2836,6 @@ dependencies = [ "serde", ] -[[package]] -name = "mysql" -version = "25.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6ad644efb545e459029b1ffa7c969d830975bd76906820913247620df10050b" -dependencies = [ - "bufstream", - "bytes", - "crossbeam", - "flate2", - "io-enum", - "libc", - "lru", - "mysql_common", - "named_pipe", - "native-tls", - "pem", - "percent-encoding", - "serde", - "serde_json", - "socket2 0.5.10", - "twox-hash", - "url", -] - -[[package]] -name = "mysql-common-derive" -version = "0.31.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63c3512cf11487168e0e9db7157801bf5273be13055a9cc95356dc9e0035e49c" -dependencies = [ - "darling 0.20.11", - "heck", - "num-bigint", - "proc-macro-crate", - "proc-macro-error2", - "proc-macro2", - "quote", - "syn 2.0.117", - "termcolor", - "thiserror 1.0.69", -] - -[[package]] -name = "mysql_common" -version = "0.32.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc" -dependencies = [ - "base64 0.21.7", - "bigdecimal", - "bindgen", - "bitflags", - "bitvec", - "btoi", - "byteorder", - "bytes", - "cc", - "cmake", - "crc32fast", - "flate2", - "frunk", - "lazy_static", - "mysql-common-derive", - "num-bigint", - "num-traits", - "rand 0.8.5", - "regex", - "rust_decimal", - "saturating", - "serde", - "serde_json", - "sha1", - "sha2", - "smallvec", - "subprocess", - "thiserror 1.0.69", - "time", - "uuid", - "zstd", -] - -[[package]] -name = "named_pipe" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b" -dependencies = [ - "winapi", -] - -[[package]] -name = "native-tls" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "neli" version = "0.7.4" @@ -3176,16 +2865,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nonempty" version = "0.7.0" @@ -3225,6 +2904,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -3278,6 +2973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3307,50 +3003,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "openssl" -version = "0.10.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "openssl-sys" -version = "0.9.113" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "owo-colors" version = "4.3.0" @@ -3445,13 +3103,12 @@ dependencies = [ ] [[package]] -name = "pem" -version = "3.0.6" +name = "pem-rfc7468" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ - "base64 0.22.1", - "serde_core", + "base64ct", ] [[package]] @@ -3541,6 +3198,27 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -3765,26 +3443,6 @@ dependencies = [ "prost", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "quickcheck" version = "1.1.0" @@ -3809,7 +3467,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -3847,7 +3505,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -3873,44 +3531,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - -[[package]] -name = "r2d2_mysql" -version = "25.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93963fe09ca35b0311d089439e944e42a6cb39bf8ea323782ddb31240ba2ae87" -dependencies = [ - "mysql", - "r2d2", -] - -[[package]] -name = "r2d2_sqlite" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5576df16239e4e422c4835c8ed00be806d4491855c7847dba60b7aa8408b469b" -dependencies = [ - "r2d2", - "rusqlite", - "uuid", -] - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "rand" version = "0.8.5" @@ -4080,15 +3700,6 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - [[package]] name = "reqwest" version = "0.13.2" @@ -4156,42 +3767,23 @@ dependencies = [ ] [[package]] -name = "rkyv" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "rsqlite-vfs" -version = "0.1.0" +name = "rsa" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "hashbrown 0.16.1", - "thiserror 2.0.18", + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", ] [[package]] @@ -4253,38 +3845,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rusqlite" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", - "sqlite-wasm-rs", -] - -[[package]] -name = "rust_decimal" -version = "1.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", - "wasm-bindgen", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -4417,12 +3977,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "saturating" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" - [[package]] name = "schannel" version = "0.1.29" @@ -4432,15 +3986,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - [[package]] name = "schemars" version = "0.9.0" @@ -4471,12 +4016,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "3.7.0" @@ -4714,65 +4253,259 @@ dependencies = [ ] [[package]] -name = "simd-adler32" -version = "0.3.9" +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] [[package]] -name = "simdutf8" -version = "0.1.5" +name = "sqlx" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] [[package]] -name = "siphasher" -version = "1.0.2" +name = "sqlx-core" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener 5.4.1", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.14.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", +] [[package]] -name = "slab" -version = "0.4.12" +name = "sqlx-macros" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.117", +] [[package]] -name = "smallvec" -version = "1.15.1" +name = "sqlx-macros-core" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.117", + "tokio", + "url", +] [[package]] -name = "socket2" -version = "0.5.10" +name = "sqlx-mysql" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ - "libc", - "windows-sys 0.52.0", + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami", ] [[package]] -name = "socket2" -version = "0.6.3" +name = "sqlx-postgres" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ - "libc", - "windows-sys 0.61.2", + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera 0.8.0", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami", ] [[package]] -name = "sqlite-wasm-rs" -version = "0.5.2" +name = "sqlx-sqlite" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4206ed3a67690b9c29b77d728f6acc3ce78f16bf846d83c94f76400320181b" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.18", + "tracing", + "url", ] [[package]] @@ -4787,6 +4520,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4816,16 +4560,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "subprocess" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c56e8662b206b9892d7a5a3f2ecdbcb455d3d6b259111373b7e08b8055158a8" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "subtle" version = "2.6.1" @@ -4916,12 +4650,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "tdyne-peer-id" version = "1.0.2" @@ -4952,15 +4680,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.4.4" @@ -4989,7 +4708,7 @@ dependencies = [ "bytes", "docker_credential", "either", - "etcetera", + "etcetera 0.11.0", "ferroid", "futures", "http", @@ -5135,16 +4854,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -5314,7 +5033,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.3", + "socket2", "sync_wrapper", "tokio", "tokio-stream", @@ -5882,17 +5601,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "rand 0.8.5", - "static_assertions", -] - [[package]] name = "typenum" version = "1.19.0" @@ -5908,6 +5616,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -5920,6 +5634,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.2" @@ -6016,7 +5745,6 @@ checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", - "rand 0.10.1", "wasm-bindgen", ] @@ -6087,6 +5815,12 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.118" @@ -6096,7 +5830,6 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", - "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] @@ -6206,6 +5939,34 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6316,6 +6077,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -6358,6 +6128,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -6397,6 +6182,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6415,6 +6206,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -6433,6 +6230,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -6463,6 +6266,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6481,6 +6290,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6499,6 +6314,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6517,6 +6338,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -6641,15 +6468,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - [[package]] name = "xattr" version = "1.6.1" diff --git a/Containerfile b/Containerfile index e926a5202..562acbdfd 100644 --- a/Containerfile +++ b/Containerfile @@ -6,7 +6,7 @@ FROM docker.io/library/rust:trixie AS chef WORKDIR /tmp RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash -RUN cargo binstall --no-confirm cargo-chef cargo-nextest +RUN cargo binstall --no-confirm --locked cargo-chef cargo-nextest ## Tester Image FROM docker.io/library/rust:slim-trixie AS tester @@ -14,7 +14,7 @@ WORKDIR /tmp RUN apt-get update; apt-get install -y curl sqlite3; apt-get autoclean RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash -RUN cargo binstall --no-confirm cargo-nextest +RUN cargo binstall --no-confirm --locked cargo-nextest COPY ./share/ /app/share/torrust RUN mkdir -p /app/share/torrust/default/database/; \ @@ -72,7 +72,9 @@ RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no- RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/torrust-tracker /app/bin/torrust-tracker -RUN mkdir /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-tracker | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 +RUN mkdir -p /app/lib/; \ + libz_path="$(ldd /app/bin/torrust-tracker | awk '/libz\.so\.1/ { print $3; exit }')"; \ + if [ -n "$libz_path" ]; then cp -l "$(realpath "$libz_path")" /app/lib/libz.so.1; fi RUN chown -R root:root /app; chmod -R u=rw,go=r,a+X /app; chmod -R a+x /app/bin # Extract and Test (release) @@ -86,7 +88,9 @@ RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no- RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json RUN mkdir -p /app/bin/; cp -l /test/src/target/release/torrust-tracker /app/bin/torrust-tracker; cp -l /test/src/target/release/http_health_check /app/bin/http_health_check -RUN mkdir -p /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-tracker | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 +RUN mkdir -p /app/lib/; \ + libz_path="$(ldd /app/bin/torrust-tracker | awk '/libz\.so\.1/ { print $3; exit }')"; \ + if [ -n "$libz_path" ]; then cp -l "$(realpath "$libz_path")" /app/lib/libz.so.1; fi RUN chown -R root:root /app; chmod -R u=rw,go=r,a+X /app; chmod -R a+x /app/bin diff --git a/contrib/dev-tools/qa/run-before-after-db-benchmark.py b/contrib/dev-tools/qa/run-before-after-db-benchmark.py new file mode 100755 index 000000000..f9a62a20b --- /dev/null +++ b/contrib/dev-tools/qa/run-before-after-db-benchmark.py @@ -0,0 +1,731 @@ +#!/usr/bin/env python3 + +import argparse +import atexit +import concurrent.futures +import hashlib +import http.client +import json +import math +import os +import shutil +import signal +import socket +import statistics +import subprocess +import tempfile +import threading +import time +import urllib.parse +import urllib.request +from pathlib import Path + + +ROOT_DIR = Path(__file__).resolve().parents[3] +DEFAULT_AFTER_REPO = Path(os.environ.get("TORRUST_TRACKER_AFTER_REPO", str(ROOT_DIR))) +DEFAULT_BEFORE_REPO = Path( + os.environ.get("TORRUST_TRACKER_BEFORE_REPO", str(ROOT_DIR.parent / "torrust-tracker-before-bench")) +) +API_TOKEN = "CodexBenchmarkToken" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Benchmark before/after tracker persistence behavior across sqlite3, mysql, and postgresql." + ) + parser.add_argument("--before-repo", type=Path, default=DEFAULT_BEFORE_REPO) + parser.add_argument("--after-repo", type=Path, default=DEFAULT_AFTER_REPO) + parser.add_argument("--dbs", nargs="+", choices=("sqlite3", "mysql", "postgresql"), default=["sqlite3", "mysql", "postgresql"]) + parser.add_argument("--mysql-version", default="8.4") + parser.add_argument("--postgres-version", default="16") + parser.add_argument("--ops", type=int, default=200) + parser.add_argument("--reload-iterations", type=int, default=30) + parser.add_argument("--concurrency", type=int, default=16) + parser.add_argument("--skip-build", action="store_true") + parser.add_argument("--json-output", type=Path) + return parser.parse_args() + + +def run_command(*args: str, cwd: Path | None = None, env: dict[str, str] | None = None, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + args, + cwd=cwd, + env=env, + text=True, + capture_output=True, + check=check, + ) + + +def choose_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def wait_for_http_ok(url: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + last_error = None + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as response: + if response.status == 200: + return + except Exception as err: # noqa: BLE001 + last_error = err + time.sleep(0.25) + raise RuntimeError(f"Timed out waiting for {url!r}: {last_error}") + + +def docker_exec(container: str, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return run_command("docker", "exec", container, *args, check=check) + + +def wait_for_mysql(container: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + result = docker_exec( + container, + "sh", + "-lc", + "mysqladmin ping -h 127.0.0.1 --password=test --silent", + check=False, + ) + if result.returncode == 0: + return + time.sleep(1) + raise RuntimeError(f"Timed out waiting for MySQL container {container!r}") + + +def wait_for_postgres(container: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + result = docker_exec( + container, + "sh", + "-lc", + "pg_isready -U postgres -d torrust_tracker", + check=False, + ) + if result.returncode == 0: + return + time.sleep(1) + raise RuntimeError(f"Timed out waiting for PostgreSQL container {container!r}") + + +def build_tracker_binary(repo_path: Path) -> None: + run_command("cargo", "build", "--release", "--bin", "torrust-tracker", cwd=repo_path) + + +def start_database( + driver: str, + workspace: Path, + mysql_version: str, + postgres_version: str, + cleanup_items: list[tuple[str, str]], +) -> str: + if driver == "sqlite3": + return str(workspace / "tracker.sqlite3.db") + + if driver == "mysql": + host_port = choose_free_port() + container = f"torrust-bench-mysql-{os.getpid()}-{host_port}" + run_command( + "docker", + "run", + "-d", + "--rm", + "--name", + container, + "-e", + "MYSQL_ROOT_HOST=%", + "-e", + "MYSQL_ROOT_PASSWORD=test", + "-e", + "MYSQL_DATABASE=torrust_tracker", + "-p", + f"127.0.0.1:{host_port}:3306", + f"mysql:{mysql_version}", + "--default-authentication-plugin=mysql_native_password", + ) + cleanup_items.append(("container", container)) + wait_for_mysql(container, 180) + return f"mysql://root:test@127.0.0.1:{host_port}/torrust_tracker" + + host_port = choose_free_port() + container = f"torrust-bench-postgres-{os.getpid()}-{host_port}" + run_command( + "docker", + "run", + "-d", + "--rm", + "--name", + container, + "-e", + "POSTGRES_PASSWORD=test", + "-e", + "POSTGRES_USER=postgres", + "-e", + "POSTGRES_DB=torrust_tracker", + "-p", + f"127.0.0.1:{host_port}:5432", + f"postgres:{postgres_version}", + ) + cleanup_items.append(("container", container)) + wait_for_postgres(container, 180) + return f"postgresql://postgres:test@127.0.0.1:{host_port}/torrust_tracker" + + +def write_tracker_config(workspace: Path, driver: str, database_path: str, tracker_port: int, api_port: int, health_port: int) -> Path: + config_path = workspace / "tracker.toml" + config_path.write_text( + "\n".join( + [ + '[metadata]', + 'app = "torrust-tracker"', + 'purpose = "configuration"', + 'schema_version = "2.0.0"', + "", + "[logging]", + 'threshold = "error"', + "", + "[core]", + "listed = false", + "private = false", + "", + "[core.database]", + f'driver = "{driver}"', + f'path = "{database_path}"', + "", + "[[http_trackers]]", + f'bind_address = "127.0.0.1:{tracker_port}"', + "", + "[http_api]", + f'bind_address = "127.0.0.1:{api_port}"', + "", + "[http_api.access_tokens]", + f'admin = "{API_TOKEN}"', + "", + "[health_check_api]", + f'bind_address = "127.0.0.1:{health_port}"', + "", + ] + ), + encoding="utf-8", + ) + return config_path + + +def start_tracker(repo_path: Path, config_path: Path, log_path: Path, cleanup_items: list[tuple[str, str]], health_port: int) -> tuple[subprocess.Popen, float]: + binary = repo_path / "target" / "release" / "torrust-tracker" + env = os.environ.copy() + env["TORRUST_TRACKER_CONFIG_TOML_PATH"] = str(config_path) + log_file = log_path.open("w", encoding="utf-8") + started_at = time.perf_counter() + process = subprocess.Popen( + [str(binary)], + cwd=repo_path, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + ) + cleanup_items.append(("process", str(process.pid))) + wait_for_http_ok(f"http://127.0.0.1:{health_port}/health_check", 30) + startup_ms = (time.perf_counter() - started_at) * 1000.0 + return process, startup_ms + + +def stop_tracker(process: subprocess.Popen, cleanup_items: list[tuple[str, str]]) -> None: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + pid = str(process.pid) + for index, item in enumerate(cleanup_items): + if item == ("process", pid): + cleanup_items.pop(index) + break + + +def cleanup(workspace: Path | None, cleanup_items: list[tuple[str, str]]) -> None: + while cleanup_items: + kind, value = cleanup_items.pop() + if kind == "container": + subprocess.run(["docker", "rm", "-f", value], text=True, capture_output=True) + elif kind == "process": + try: + os.kill(int(value), signal.SIGTERM) + except ProcessLookupError: + pass + if workspace and workspace.exists(): + shutil.rmtree(workspace, ignore_errors=True) + + +class ThreadLocalHttpClient: + def __init__(self, timeout: float = 5.0) -> None: + self.timeout = timeout + self.local = threading.local() + + def _connections(self) -> dict[tuple[str, str, int], http.client.HTTPConnection]: + connections = getattr(self.local, "connections", None) + if connections is None: + connections = {} + self.local.connections = connections + return connections + + def request(self, method: str, url: str, headers: dict[str, str] | None = None, body: bytes | None = None) -> tuple[int, bytes]: + parsed = urllib.parse.urlsplit(url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + key = (parsed.scheme, parsed.hostname or "127.0.0.1", port) + connections = self._connections() + connection = connections.get(key) + + if connection is None: + if parsed.scheme == "https": + connection = http.client.HTTPSConnection(key[1], key[2], timeout=self.timeout) + else: + connection = http.client.HTTPConnection(key[1], key[2], timeout=self.timeout) + connections[key] = connection + + try: + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + payload = response.read() + return response.status, payload + except Exception: # noqa: BLE001 + try: + connection.close() + except Exception: # noqa: BLE001 + pass + connections.pop(key, None) + if parsed.scheme == "https": + connection = http.client.HTTPSConnection(key[1], key[2], timeout=self.timeout) + else: + connection = http.client.HTTPConnection(key[1], key[2], timeout=self.timeout) + connections[key] = connection + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + payload = response.read() + return response.status, payload + + +def http_request(method: str, url: str, headers: dict[str, str] | None = None, body: bytes | None = None) -> tuple[int, bytes]: + request = urllib.request.Request(url, data=body, headers=headers or {}, method=method) + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, response.read() + + +def percentile(sorted_values: list[float], fraction: float) -> float: + if not sorted_values: + return 0.0 + index = max(0, math.ceil(len(sorted_values) * fraction) - 1) + return sorted_values[index] + + +def summarize_latencies(latencies_ms: list[float], total_seconds: float) -> dict[str, float]: + ordered = sorted(latencies_ms) + return { + "count": float(len(latencies_ms)), + "total_ms": total_seconds * 1000.0, + "ops_per_sec": len(latencies_ms) / total_seconds if total_seconds > 0 else 0.0, + "mean_ms": statistics.fmean(latencies_ms) if latencies_ms else 0.0, + "median_ms": statistics.median(latencies_ms) if latencies_ms else 0.0, + "p95_ms": percentile(ordered, 0.95), + "min_ms": ordered[0] if ordered else 0.0, + "max_ms": ordered[-1] if ordered else 0.0, + } + + +def benchmark_operations(label: str, functions: list[callable], concurrency: int) -> dict[str, object]: + latencies_ms: list[float] = [] + started_at = time.perf_counter() + + if concurrency == 1: + for operation in functions: + op_started_at = time.perf_counter() + operation() + latencies_ms.append((time.perf_counter() - op_started_at) * 1000.0) + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [] + for operation in functions: + futures.append(executor.submit(run_timed_operation, operation)) + for future in concurrent.futures.as_completed(futures): + latencies_ms.append(future.result()) + + total_seconds = time.perf_counter() - started_at + return { + "label": label, + "stats": summarize_latencies(latencies_ms, total_seconds), + } + + +def run_timed_operation(operation: callable) -> float: + started_at = time.perf_counter() + operation() + return (time.perf_counter() - started_at) * 1000.0 + + +def sha1_bytes(seed: str) -> bytes: + return hashlib.sha1(seed.encode("utf-8")).digest() + + +def info_hash_bytes(prefix: str, index: int) -> bytes: + return sha1_bytes(f"{prefix}-{index}") + + +def info_hash_hex(prefix: str, index: int) -> str: + return info_hash_bytes(prefix, index).hex() + + +def peer_id_bytes(prefix: str, index: int) -> bytes: + prefix_id = int.from_bytes(hashlib.sha1(prefix.encode("utf-8")).digest()[:2], "big") % 100 + return f"-CD{prefix_id:02d}-{index:014d}".encode("ascii") + + +def auth_key_value(prefix: str, index: int) -> str: + return hashlib.sha256(f"{prefix}-{index}".encode("utf-8")).hexdigest()[:32] + + +def authorize_headers(content_type: str | None = None) -> dict[str, str]: + headers = {"Authorization": f"Bearer {API_TOKEN}"} + if content_type is not None: + headers["Content-Type"] = content_type + return headers + + +def assert_status(status: int, payload: bytes, expected: set[int], url: str) -> None: + if status not in expected: + snippet = payload.decode("utf-8", errors="replace")[:300] + raise RuntimeError(f"Unexpected HTTP status {status} for {url}: {snippet}") + + +def post_whitelist(client: ThreadLocalHttpClient, api_origin: str, info_hash: str) -> None: + url = f"{api_origin}/api/v1/whitelist/{info_hash}" + status, payload = http_request("POST", url, headers=authorize_headers()) + assert_status(status, payload, {200, 201, 204}, url) + + +def reload_whitelist(client: ThreadLocalHttpClient, api_origin: str) -> None: + url = f"{api_origin}/api/v1/whitelist/reload" + status, payload = http_request("GET", url, headers=authorize_headers()) + assert_status(status, payload, {200}, url) + + +def post_auth_key(client: ThreadLocalHttpClient, api_origin: str, key: str) -> None: + url = f"{api_origin}/api/v1/keys" + body = json.dumps({"key": key, "seconds_valid": 3600}).encode("utf-8") + status, payload = http_request("POST", url, headers=authorize_headers("application/json"), body=body) + assert_status(status, payload, {200, 201, 204}, url) + + +def reload_keys(client: ThreadLocalHttpClient, api_origin: str) -> None: + url = f"{api_origin}/api/v1/keys/reload" + status, payload = http_request("GET", url, headers=authorize_headers()) + assert_status(status, payload, {200}, url) + + +def announce_started_then_completed(client: ThreadLocalHttpClient, tracker_origin: str, prefix: str, index: int) -> None: + info_hash = info_hash_bytes(prefix, index) + peer_id = peer_id_bytes(prefix, index) + query_started = build_announce_query(info_hash, peer_id, 1, "started") + query_completed = build_announce_query(info_hash, peer_id, 0, "completed") + + started_url = f"{tracker_origin}/announce?{query_started}" + status, payload = http_request("GET", started_url) + assert_status(status, payload, {200}, started_url) + + completed_url = f"{tracker_origin}/announce?{query_completed}" + status, payload = http_request("GET", completed_url) + assert_status(status, payload, {200}, completed_url) + + +def build_announce_query(info_hash: bytes, peer_id: bytes, left: int, event: str) -> str: + return "&".join( + [ + f"info_hash={urllib.parse.quote_from_bytes(info_hash)}", + "peer_addr=192.168.1.88", + f"peer_id={urllib.parse.quote_from_bytes(peer_id)}", + "port=17548", + "uploaded=0", + "downloaded=0", + f"left={left}", + "compact=0", + f"event={event}", + ] + ) + + +def decode_bencode(data: bytes, offset: int = 0): + token = data[offset : offset + 1] + if token == b"i": + end = data.index(b"e", offset) + return int(data[offset + 1 : end]), end + 1 + if token == b"l": + offset += 1 + items = [] + while data[offset : offset + 1] != b"e": + item, offset = decode_bencode(data, offset) + items.append(item) + return items, offset + 1 + if token == b"d": + offset += 1 + mapping = {} + while data[offset : offset + 1] != b"e": + key, offset = decode_bencode(data, offset) + value, offset = decode_bencode(data, offset) + mapping[key] = value + return mapping, offset + 1 + if token.isdigit(): + colon = data.index(b":", offset) + size = int(data[offset:colon]) + start = colon + 1 + end = start + size + return data[start:end], end + raise ValueError(f"Unexpected bencode token at offset {offset}: {token!r}") + + +def scrape_tracker(tracker_origin: str, info_hash: bytes) -> dict[bytes, object]: + url = f"{tracker_origin}/scrape?info_hash={urllib.parse.quote_from_bytes(info_hash)}" + with urllib.request.urlopen(url, timeout=5) as response: + if response.status != 200: + raise RuntimeError(f"Unexpected HTTP status {response.status} for {url}") + payload = response.read() + decoded, end_offset = decode_bencode(payload) + if end_offset != len(payload): + raise RuntimeError("Unexpected trailing bytes in scrape response") + return decoded + + +def verify_persisted_download(client: ThreadLocalHttpClient, tracker_origin: str, prefix: str, index: int) -> None: + info_hash = info_hash_bytes(prefix, index) + deadline = time.time() + 10 + last_downloaded = None + + while time.time() < deadline: + payload = scrape_tracker(tracker_origin, info_hash) + files = payload.get(b"files") + if isinstance(files, dict): + stats = files.get(info_hash) + if isinstance(stats, dict): + last_downloaded = stats.get(b"downloaded") + if last_downloaded == 1: + return + time.sleep(0.25) + + raise RuntimeError(f"Expected downloaded=1 for {info_hash.hex()}, got {last_downloaded!r}") + + +def warm_up(client: ThreadLocalHttpClient, api_origin: str, tracker_origin: str) -> None: + reload_whitelist(client, api_origin) + reload_keys(client, api_origin) + announce_started_then_completed(client, tracker_origin, "warmup-announce", 0) + + +def run_suite( + variant: str, + repo_path: Path, + driver: str, + mysql_version: str, + postgres_version: str, + ops: int, + reload_iterations: int, + concurrency: int, +) -> dict[str, object]: + workspace = Path(tempfile.mkdtemp(prefix=f"torrust-benchmark-{variant}-{driver}-")) + cleanup_items: list[tuple[str, str]] = [] + atexit.register(cleanup, workspace, cleanup_items) + + tracker_port = choose_free_port() + api_port = choose_free_port() + health_port = choose_free_port() + database_path = start_database(driver, workspace, mysql_version, postgres_version, cleanup_items) + config_path = write_tracker_config(workspace, driver, database_path, tracker_port, api_port, health_port) + log_path = workspace / "tracker.log" + + process, startup_empty_ms = start_tracker(repo_path, config_path, log_path, cleanup_items, health_port) + client = ThreadLocalHttpClient() + api_origin = f"http://127.0.0.1:{api_port}" + tracker_origin = f"http://127.0.0.1:{tracker_port}" + + try: + warm_up(client, api_origin, tracker_origin) + + results: dict[str, object] = { + "variant": variant, + "repo_path": str(repo_path), + "driver": driver, + "startup_empty_ms": startup_empty_ms, + "workloads": {}, + "log_path": str(log_path), + } + + announce_seq_ops = [ + lambda index=index: announce_started_then_completed(client, tracker_origin, "announce-seq", index) + for index in range(ops) + ] + results["workloads"]["announce_lifecycle_seq"] = benchmark_operations( + "announce_lifecycle_seq", announce_seq_ops, 1 + ) + + announce_conc_ops = [ + lambda index=index: announce_started_then_completed(client, tracker_origin, "announce-conc", index) + for index in range(ops) + ] + results["workloads"]["announce_lifecycle_concurrent"] = benchmark_operations( + "announce_lifecycle_concurrent", announce_conc_ops, concurrency + ) + + whitelist_seq_ops = [ + lambda index=index: post_whitelist(client, api_origin, info_hash_hex("whitelist-seq", index)) + for index in range(ops) + ] + results["workloads"]["whitelist_add_seq"] = benchmark_operations("whitelist_add_seq", whitelist_seq_ops, 1) + + whitelist_conc_ops = [ + lambda index=index: post_whitelist(client, api_origin, info_hash_hex("whitelist-conc", index)) + for index in range(ops) + ] + results["workloads"]["whitelist_add_concurrent"] = benchmark_operations( + "whitelist_add_concurrent", whitelist_conc_ops, concurrency + ) + + whitelist_reload_ops = [lambda: reload_whitelist(client, api_origin) for _ in range(reload_iterations)] + results["workloads"]["whitelist_reload"] = benchmark_operations("whitelist_reload", whitelist_reload_ops, 1) + + key_seq_ops = [ + lambda index=index: post_auth_key(client, api_origin, auth_key_value("bench-key-seq", index)) + for index in range(ops) + ] + results["workloads"]["auth_key_add_seq"] = benchmark_operations("auth_key_add_seq", key_seq_ops, 1) + + key_conc_ops = [ + lambda index=index: post_auth_key(client, api_origin, auth_key_value("bench-key-conc", index)) + for index in range(ops) + ] + results["workloads"]["auth_key_add_concurrent"] = benchmark_operations( + "auth_key_add_concurrent", key_conc_ops, concurrency + ) + + key_reload_ops = [lambda: reload_keys(client, api_origin) for _ in range(reload_iterations)] + results["workloads"]["auth_key_reload"] = benchmark_operations("auth_key_reload", key_reload_ops, 1) + + stop_tracker(process, cleanup_items) + process, startup_populated_ms = start_tracker(repo_path, config_path, log_path, cleanup_items, health_port) + results["startup_populated_ms"] = startup_populated_ms + reload_whitelist(client, api_origin) + reload_keys(client, api_origin) + + stop_tracker(process, cleanup_items) + return results + finally: + if process.poll() is None: + stop_tracker(process, cleanup_items) + cleanup(workspace, cleanup_items) + + +def compare_results(results: list[dict[str, object]]) -> list[dict[str, object]]: + indexed = {(entry["driver"], entry["variant"]): entry for entry in results} + comparisons: list[dict[str, object]] = [] + + for driver in sorted({entry["driver"] for entry in results}): + before = indexed[(driver, "before")] + after = indexed[(driver, "after")] + driver_comparison: dict[str, object] = { + "driver": driver, + "startup_empty_speedup": before["startup_empty_ms"] / after["startup_empty_ms"], + "startup_populated_speedup": before["startup_populated_ms"] / after["startup_populated_ms"], + "workloads": {}, + } + + before_workloads = before["workloads"] + after_workloads = after["workloads"] + for workload_name in before_workloads: + before_ops = before_workloads[workload_name]["stats"]["ops_per_sec"] + after_ops = after_workloads[workload_name]["stats"]["ops_per_sec"] + before_p95 = before_workloads[workload_name]["stats"]["p95_ms"] + after_p95 = after_workloads[workload_name]["stats"]["p95_ms"] + driver_comparison["workloads"][workload_name] = { + "ops_per_sec_speedup": after_ops / before_ops if before_ops else 0.0, + "p95_latency_improvement": before_p95 / after_p95 if after_p95 else 0.0, + } + comparisons.append(driver_comparison) + + return comparisons + + +def print_summary(results: list[dict[str, object]], comparisons: list[dict[str, object]]) -> None: + print("") + print("Startup") + print("variant driver empty_ms populated_ms") + for entry in results: + print( + f"{entry['variant']:<8} {entry['driver']:<11} " + f"{entry['startup_empty_ms']:>8.1f} {entry['startup_populated_ms']:>13.1f}" + ) + + print("") + print("Workloads (ops/s, p95 ms)") + print("variant driver workload ops_per_sec p95_ms") + for entry in results: + for workload_name, workload in entry["workloads"].items(): + print( + f"{entry['variant']:<8} {entry['driver']:<11} {workload_name:<30} " + f"{workload['stats']['ops_per_sec']:>11.2f} {workload['stats']['p95_ms']:>11.2f}" + ) + + print("") + print("After vs Before Speedup") + print("driver workload ops/s_x p95_improvement_x") + for driver_comparison in comparisons: + for workload_name, workload in driver_comparison["workloads"].items(): + print( + f"{driver_comparison['driver']:<11} {workload_name:<30} " + f"{workload['ops_per_sec_speedup']:>7.2f} {workload['p95_latency_improvement']:>18.2f}" + ) + + +def main() -> int: + args = parse_args() + + if not args.skip_build: + build_tracker_binary(args.before_repo) + build_tracker_binary(args.after_repo) + + results: list[dict[str, object]] = [] + suites = [("before", args.before_repo), ("after", args.after_repo)] + for driver in args.dbs: + for variant, repo_path in suites: + print(f"Running {variant} benchmark on {driver}...", flush=True) + result = run_suite( + variant=variant, + repo_path=repo_path, + driver=driver, + mysql_version=args.mysql_version, + postgres_version=args.postgres_version, + ops=args.ops, + reload_iterations=args.reload_iterations, + concurrency=args.concurrency, + ) + results.append(result) + + comparisons = compare_results(results) + payload = {"results": results, "comparisons": comparisons} + + if args.json_output is not None: + args.json_output.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + print_summary(results, comparisons) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/contrib/dev-tools/qa/run-db-compatibility-matrix.sh b/contrib/dev-tools/qa/run-db-compatibility-matrix.sh new file mode 100755 index 000000000..e5e2bead6 --- /dev/null +++ b/contrib/dev-tools/qa/run-db-compatibility-matrix.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +POSTGRES_VERSIONS_STRING="${POSTGRES_VERSIONS:-14 15 16 17}" +MYSQL_VERSIONS_STRING="${MYSQL_VERSIONS:-8.0 8.4}" + +read -r -a POSTGRES_VERSIONS <<< "$POSTGRES_VERSIONS_STRING" +read -r -a MYSQL_VERSIONS <<< "$MYSQL_VERSIONS_STRING" + +run_step() { + echo + echo "==> $*" + "$@" +} + +run_step bash -lc "cd '$ROOT_DIR' && cargo check --workspace --all-targets" + +echo +echo "==> SQLite runtime version" +sqlite3 --version + +run_step bash -lc "cd '$ROOT_DIR' && cargo test -p torrust-tracker-configuration postgresql_user_password" +run_step bash -lc "cd '$ROOT_DIR' && cargo test -p bittorrent-http-tracker-protocol saturate_large_download_counts" +run_step bash -lc "cd '$ROOT_DIR' && cargo test -p torrust-udp-tracker-server saturate_large_download_counts_for_udp_protocol" +run_step bash -lc "cd '$ROOT_DIR' && cargo test -p bittorrent-tracker-core run_sqlite_driver_tests -- --nocapture" + +for version in "${MYSQL_VERSIONS[@]}"; do + echo + echo "==> MySQL compatibility test on ${version}" + docker pull "mysql:${version}" + TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST=1 \ + TORRUST_TRACKER_CORE_MYSQL_DRIVER_IMAGE_TAG="${version}" \ + bash -lc "cd '$ROOT_DIR' && cargo test -p bittorrent-tracker-core run_mysql_driver_tests -- --nocapture" +done + +for version in "${POSTGRES_VERSIONS[@]}"; do + echo + echo "==> PostgreSQL compatibility test on ${version}" + docker pull "postgres:${version}" + TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST=1 \ + TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG="${version}" \ + bash -lc "cd '$ROOT_DIR' && cargo test -p bittorrent-tracker-core run_postgres_driver_tests -- --nocapture" +done + +echo +echo "Database compatibility matrix finished successfully." diff --git a/contrib/dev-tools/qa/run-qbittorrent-e2e.py b/contrib/dev-tools/qa/run-qbittorrent-e2e.py new file mode 100755 index 000000000..8dcc85274 --- /dev/null +++ b/contrib/dev-tools/qa/run-qbittorrent-e2e.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python3 + +import argparse +import atexit +import base64 +import hashlib +import json +import os +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +ROOT_DIR = Path(__file__).resolve().parents[3] +TRACKER_BINARY = ROOT_DIR / "target" / "debug" / "torrust-tracker" +DEFAULT_QBITTORRENT_IMAGE = "qbittorrentofficial/qbittorrent-nox:latest" +QBITTORRENT_PASSWORD = "codex-pass" +QBITTORRENT_USERNAME = "admin" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run a qBittorrent end-to-end test against the tracker.") + parser.add_argument("--db-driver", choices=("sqlite3", "mysql", "postgresql"), default="postgresql") + parser.add_argument("--protocol", choices=("http", "udp"), default="http") + parser.add_argument("--mysql-version", default="8.4") + parser.add_argument("--postgres-version", default="16") + parser.add_argument("--qbittorrent-image", default=DEFAULT_QBITTORRENT_IMAGE) + parser.add_argument("--timeout-seconds", type=int, default=180) + parser.add_argument("--keep-artifacts", action="store_true") + return parser.parse_args() + + +def run_command(*args: str, cwd: Path | None = None, env: dict[str, str] | None = None, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + args, + cwd=cwd or ROOT_DIR, + env=env, + text=True, + capture_output=True, + check=check, + ) + + +def choose_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def wait_for_http_ok(url: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + last_error = None + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=3) as response: + if response.status == 200: + return + except Exception as err: # noqa: BLE001 + last_error = err + time.sleep(1) + raise RuntimeError(f"Timed out waiting for {url!r}: {last_error}") + + +def docker_exec(container: str, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return run_command("docker", "exec", container, *args, check=check) + + +def docker_port(container: str, container_port: int, protocol: str = "tcp") -> int: + result = run_command("docker", "port", container, f"{container_port}/{protocol}") + _, port = result.stdout.strip().rsplit(":", 1) + return int(port) + + +def wait_for_mysql(container: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + result = docker_exec( + container, + "sh", + "-lc", + "mysqladmin ping -h 127.0.0.1 --password=test --silent", + check=False, + ) + if result.returncode == 0: + return + time.sleep(1) + raise RuntimeError(f"Timed out waiting for MySQL container {container!r}") + + +def wait_for_postgres(container: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + result = docker_exec( + container, + "sh", + "-lc", + "pg_isready -U postgres -d torrust_tracker", + check=False, + ) + if result.returncode == 0: + return + time.sleep(1) + raise RuntimeError(f"Timed out waiting for PostgreSQL container {container!r}") + + +def pbkdf2_password(password: str) -> str: + salt = os.urandom(16) + digest = hashlib.pbkdf2_hmac("sha512", password.encode(), salt, 100_000) + return f"@ByteArray({base64.b64encode(salt).decode()}:{base64.b64encode(digest).decode()})" + + +def write_qbittorrent_config(config_root: Path, peer_port: int) -> None: + config_dir = config_root / "qBittorrent" / "config" + config_dir.mkdir(parents=True, exist_ok=True) + config = ( + "[BitTorrent]\n" + "Session\\AddTorrentStopped=false\n" + "Session\\DefaultSavePath=/downloads\n" + f"Session\\Port={peer_port}\n" + "Session\\TempPath=/downloads/temp\n" + "[Preferences]\n" + "WebUI\\LocalHostAuth=false\n" + "WebUI\\Port=8080\n" + f'WebUI\\Password_PBKDF2="{pbkdf2_password(QBITTORRENT_PASSWORD)}"\n' + f"WebUI\\Username={QBITTORRENT_USERNAME}\n" + ) + (config_dir / "qBittorrent.conf").write_text(config, encoding="utf-8") + + +def bencode(value) -> bytes: + if isinstance(value, int): + return f"i{value}e".encode() + if isinstance(value, bytes): + return str(len(value)).encode() + b":" + value + if isinstance(value, str): + return bencode(value.encode()) + if isinstance(value, list): + return b"l" + b"".join(bencode(item) for item in value) + b"e" + if isinstance(value, dict): + encoded_items = [] + for key in sorted(value): + encoded_items.append(bencode(key)) + encoded_items.append(bencode(value[key])) + return b"d" + b"".join(encoded_items) + b"e" + raise TypeError(f"Unsupported bencode type: {type(value)!r}") + + +def decode_bencode(data: bytes, offset: int = 0): + token = data[offset : offset + 1] + if token == b"i": + end = data.index(b"e", offset) + return int(data[offset + 1 : end]), end + 1 + if token == b"l": + offset += 1 + items = [] + while data[offset : offset + 1] != b"e": + item, offset = decode_bencode(data, offset) + items.append(item) + return items, offset + 1 + if token == b"d": + offset += 1 + mapping = {} + while data[offset : offset + 1] != b"e": + key, offset = decode_bencode(data, offset) + value, offset = decode_bencode(data, offset) + mapping[key] = value + return mapping, offset + 1 + if token.isdigit(): + colon = data.index(b":", offset) + size = int(data[offset:colon]) + start = colon + 1 + end = start + size + return data[start:end], end + raise ValueError(f"Unexpected bencode token at offset {offset}: {token!r}") + + +def build_torrent(payload_path: Path, torrent_path: Path, announce_url: str) -> bytes: + payload = payload_path.read_bytes() + piece_length = 16 * 1024 + pieces = b"".join(hashlib.sha1(payload[index : index + piece_length]).digest() for index in range(0, len(payload), piece_length)) + info = { + b"length": len(payload), + b"name": payload_path.name.encode(), + b"piece length": piece_length, + b"pieces": pieces, + } + torrent = { + b"announce": announce_url.encode(), + b"created by": b"codex-qb-e2e", + b"creation date": int(time.time()), + b"info": info, + } + torrent_path.write_bytes(bencode(torrent)) + return hashlib.sha1(bencode(info)).digest() + + +def build_tracker_binary() -> None: + run_command("cargo", "build", "--bin", "torrust-tracker") + + +def start_database(driver: str, workspace: Path, args: argparse.Namespace, cleanup_items: list[tuple[str, str]]): + if driver == "sqlite3": + db_path = workspace / "tracker.sqlite3.db" + return str(db_path) + + if driver == "mysql": + host_port = choose_free_port() + container = f"torrust-mysql-e2e-{os.getpid()}" + run_command( + "docker", + "run", + "-d", + "--rm", + "--name", + container, + "-e", + "MYSQL_ROOT_HOST=%", + "-e", + "MYSQL_ROOT_PASSWORD=test", + "-e", + "MYSQL_DATABASE=torrust_tracker", + "-p", + f"127.0.0.1:{host_port}:3306", + f"mysql:{args.mysql_version}", + "--default-authentication-plugin=mysql_native_password", + ) + cleanup_items.append(("container", container)) + wait_for_mysql(container, 60) + return f"mysql://root:test@127.0.0.1:{host_port}/torrust_tracker" + + host_port = choose_free_port() + container = f"torrust-postgres-e2e-{os.getpid()}" + run_command( + "docker", + "run", + "-d", + "--rm", + "--name", + container, + "-e", + "POSTGRES_PASSWORD=test", + "-e", + "POSTGRES_USER=postgres", + "-e", + "POSTGRES_DB=torrust_tracker", + "-p", + f"127.0.0.1:{host_port}:5432", + f"postgres:{args.postgres_version}", + ) + cleanup_items.append(("container", container)) + wait_for_postgres(container, 60) + return f"postgresql://postgres:test@127.0.0.1:{host_port}/torrust_tracker" + + +def write_tracker_config(workspace: Path, driver: str, database_path: str, http_port: int, udp_port: int, health_port: int) -> Path: + config_path = workspace / "tracker.toml" + config_path.write_text( + "\n".join( + [ + '[metadata]', + 'app = "torrust-tracker"', + 'purpose = "configuration"', + 'schema_version = "2.0.0"', + "", + "[logging]", + 'threshold = "debug"', + "", + "[core]", + "listed = false", + "private = false", + "", + "[core.database]", + f'driver = "{driver}"', + f'path = "{database_path}"', + "", + "[[udp_trackers]]", + f'bind_address = "0.0.0.0:{udp_port}"', + "", + "[[http_trackers]]", + f'bind_address = "0.0.0.0:{http_port}"', + "", + "[health_check_api]", + f'bind_address = "127.0.0.1:{health_port}"', + "", + ] + ), + encoding="utf-8", + ) + return config_path + + +def start_tracker(config_path: Path, log_path: Path, cleanup_items: list[tuple[str, str]], health_port: int) -> subprocess.Popen: + env = os.environ.copy() + env["TORRUST_TRACKER_CONFIG_TOML_PATH"] = str(config_path) + log_file = log_path.open("w", encoding="utf-8") + process = subprocess.Popen( + [str(TRACKER_BINARY)], + cwd=ROOT_DIR, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + ) + cleanup_items.append(("process", str(process.pid))) + wait_for_http_ok(f"http://127.0.0.1:{health_port}/health_check", 30) + return process + + +def qb_login(container: str) -> None: + result = docker_exec( + container, + "curl", + "-s", + "-c", + "/tmp/qb.cookies", + "--data", + f"username={QBITTORRENT_USERNAME}&password={QBITTORRENT_PASSWORD}", + "http://127.0.0.1:8080/api/v2/auth/login", + ) + if result.stdout.strip() != "Ok.": + raise RuntimeError(f"Unable to login to qBittorrent container {container!r}: {result.stdout!r} {result.stderr!r}") + + +def qb_get_json(container: str, path: str) -> object: + qb_login(container) + result = docker_exec( + container, + "curl", + "-s", + "-b", + "/tmp/qb.cookies", + f"http://127.0.0.1:8080{path}", + ) + return json.loads(result.stdout) + + +def qb_get_text(container: str, path: str) -> str: + qb_login(container) + result = docker_exec( + container, + "curl", + "-s", + "-b", + "/tmp/qb.cookies", + f"http://127.0.0.1:8080{path}", + ) + return result.stdout.strip() + + +def qb_post_form(container: str, path: str, args: list[str]) -> str: + qb_login(container) + result = docker_exec( + container, + "curl", + "-s", + "-b", + "/tmp/qb.cookies", + *args, + f"http://127.0.0.1:8080{path}", + ) + return result.stdout + + +def wait_for_qb_api(container: str, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + try: + version = qb_get_text(container, "/api/v2/app/version") + if version.startswith("v"): + return + except Exception: # noqa: BLE001 + pass + time.sleep(1) + raise RuntimeError(f"Timed out waiting for qBittorrent API in {container!r}") + + +def start_qbittorrent(name: str, image: str, config_root: Path, downloads_root: Path, shared_root: Path, peer_port: int, cleanup_items: list[tuple[str, str]]) -> str: + subprocess.run(["docker", "rm", "-f", name], text=True, capture_output=True) + run_command( + "docker", + "run", + "-d", + "--rm", + "--name", + name, + "-e", + "QBT_LEGAL_NOTICE=confirm", + "-e", + f"QBT_TORRENTING_PORT={peer_port}", + "-v", + f"{config_root}:/config", + "-v", + f"{downloads_root}:/downloads", + "-v", + f"{shared_root}:/shared", + "-p", + f"{peer_port}:{peer_port}", + "-p", + f"{peer_port}:{peer_port}/udp", + image, + ) + cleanup_items.append(("container", name)) + wait_for_qb_api(name, 30) + return name + + +def qb_add_torrent(container: str, torrent_path_in_container: str) -> None: + qb_post_form( + container, + "/api/v2/torrents/add", + [ + "-F", + f"torrents=@{torrent_path_in_container}", + "-F", + "savepath=/downloads", + "-F", + "paused=false", + "-F", + "skip_checking=false", + ], + ) + + +def qb_torrent_info(container: str, info_hash_hex: str) -> dict: + torrents = qb_get_json(container, f"/api/v2/torrents/info?hashes={info_hash_hex}") + if not torrents: + raise RuntimeError(f"Torrent {info_hash_hex} not found in {container!r}") + return torrents[0] + + +def qb_trackers(container: str, info_hash_hex: str) -> list[dict]: + trackers = qb_get_json(container, f"/api/v2/torrents/trackers?hash={info_hash_hex}") + return [tracker for tracker in trackers if isinstance(tracker, dict) and tracker.get("url")] + + +def wait_for_progress(container: str, info_hash_hex: str, expected_progress: float, timeout_seconds: int) -> dict: + deadline = time.time() + timeout_seconds + last_info = None + while time.time() < deadline: + last_info = qb_torrent_info(container, info_hash_hex) + if float(last_info.get("progress", 0.0)) >= expected_progress: + return last_info + time.sleep(1) + raise RuntimeError(f"Timed out waiting for progress {expected_progress} in {container!r}: {last_info}") + + +def wait_for_tracker_contact(container: str, info_hash_hex: str, timeout_seconds: int) -> list[dict]: + deadline = time.time() + timeout_seconds + last_trackers = None + while time.time() < deadline: + last_trackers = qb_trackers(container, info_hash_hex) + if any(tracker.get("status", 0) != 0 for tracker in last_trackers): + return last_trackers + time.sleep(1) + raise RuntimeError(f"Timed out waiting for tracker contact in {container!r}: {last_trackers}") + + +def scrape_tracker(http_port: int, info_hash: bytes) -> dict[bytes, object]: + encoded_info_hash = urllib.parse.quote_from_bytes(info_hash) + url = f"http://127.0.0.1:{http_port}/scrape?info_hash={encoded_info_hash}" + with urllib.request.urlopen(url, timeout=10) as response: + payload = response.read() + decoded, end_offset = decode_bencode(payload) + if end_offset != len(payload): + raise RuntimeError("Unexpected trailing bytes in scrape response") + return decoded + + +def cleanup(workspace: Path | None, keep_artifacts: bool, cleanup_items: list[tuple[str, str]]) -> None: + while cleanup_items: + kind, value = cleanup_items.pop() + if kind == "container": + subprocess.run(["docker", "rm", "-f", value], text=True, capture_output=True) + elif kind == "process": + try: + os.kill(int(value), signal.SIGTERM) + except ProcessLookupError: + pass + if workspace and workspace.exists() and not keep_artifacts: + shutil.rmtree(workspace, ignore_errors=True) + + +def main() -> int: + args = parse_args() + workspace = Path(tempfile.mkdtemp(prefix="torrust-qb-e2e-")) + cleanup_items: list[tuple[str, str]] = [] + atexit.register(cleanup, workspace, args.keep_artifacts, cleanup_items) + + http_port = choose_free_port() + udp_port = choose_free_port() + health_port = choose_free_port() + seeder_peer_port = choose_free_port() + leecher_peer_port = choose_free_port() + + shared_root = workspace / "shared" + seeder_downloads = workspace / "seeder-downloads" + leecher_downloads = workspace / "leecher-downloads" + seeder_config = workspace / "seeder-config" + leecher_config = workspace / "leecher-config" + + shared_root.mkdir(parents=True, exist_ok=True) + seeder_downloads.mkdir(parents=True, exist_ok=True) + leecher_downloads.mkdir(parents=True, exist_ok=True) + + write_qbittorrent_config(seeder_config, seeder_peer_port) + write_qbittorrent_config(leecher_config, leecher_peer_port) + + payload_path = shared_root / "payload.bin" + payload_path.write_bytes(os.urandom(256 * 1024)) + shutil.copy2(payload_path, seeder_downloads / payload_path.name) + + tracker_scheme = "http" if args.protocol == "http" else "udp" + tracker_port = http_port if args.protocol == "http" else udp_port + torrent_path = shared_root / "payload.torrent" + info_hash = build_torrent(payload_path, torrent_path, f"{tracker_scheme}://host.docker.internal:{tracker_port}/announce") + info_hash_hex = info_hash.hex() + + build_tracker_binary() + database_path = start_database(args.db_driver, workspace, args, cleanup_items) + tracker_config = write_tracker_config(workspace, args.db_driver, database_path, http_port, udp_port, health_port) + tracker_log = workspace / "tracker.log" + tracker_process = start_tracker(tracker_config, tracker_log, cleanup_items, health_port) + + start_qbittorrent("torrust-qb-seeder", args.qbittorrent_image, seeder_config, seeder_downloads, shared_root, seeder_peer_port, cleanup_items) + start_qbittorrent("torrust-qb-leecher", args.qbittorrent_image, leecher_config, leecher_downloads, shared_root, leecher_peer_port, cleanup_items) + + qb_add_torrent("torrust-qb-seeder", "/shared/payload.torrent") + seeder_info = wait_for_progress("torrust-qb-seeder", info_hash_hex, 1.0, args.timeout_seconds) + seeder_trackers = wait_for_tracker_contact("torrust-qb-seeder", info_hash_hex, args.timeout_seconds) + + qb_add_torrent("torrust-qb-leecher", "/shared/payload.torrent") + leecher_info = wait_for_progress("torrust-qb-leecher", info_hash_hex, 1.0, args.timeout_seconds) + leecher_trackers = wait_for_tracker_contact("torrust-qb-leecher", info_hash_hex, args.timeout_seconds) + + downloaded_path = leecher_downloads / payload_path.name + if not downloaded_path.exists(): + raise RuntimeError(f"Leecher did not create the expected payload file: {downloaded_path}") + if downloaded_path.read_bytes() != payload_path.read_bytes(): + raise RuntimeError("Leecher payload does not match the seeded payload") + + scrape_response = scrape_tracker(http_port, info_hash) + files_section = scrape_response[b"files"] + torrent_stats = files_section[info_hash] + if torrent_stats[b"complete"] < 1: + raise RuntimeError(f"Unexpected scrape complete count: {torrent_stats}") + if torrent_stats[b"downloaded"] < 1: + raise RuntimeError(f"Unexpected scrape downloaded count: {torrent_stats}") + + print("Seeder info:", json.dumps(seeder_info, indent=2, sort_keys=True)) + print("Seeder trackers:", json.dumps(seeder_trackers, indent=2, sort_keys=True)) + print("Leecher info:", json.dumps(leecher_info, indent=2, sort_keys=True)) + print("Leecher trackers:", json.dumps(leecher_trackers, indent=2, sort_keys=True)) + print( + "Tracker scrape stats:", + json.dumps( + { + "complete": torrent_stats[b"complete"], + "downloaded": torrent_stats[b"downloaded"], + "incomplete": torrent_stats[b"incomplete"], + }, + indent=2, + sort_keys=True, + ), + ) + print(f"Tracker log: {tracker_log}") + + tracker_process.terminate() + try: + tracker_process.wait(timeout=10) + except subprocess.TimeoutExpired: + tracker_process.kill() + tracker_process.wait(timeout=10) + + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as err: # noqa: BLE001 + print(err, file=sys.stderr) + raise diff --git a/docs/postgresql-adaptation-rework.md b/docs/postgresql-adaptation-rework.md new file mode 100644 index 000000000..644a304a6 --- /dev/null +++ b/docs/postgresql-adaptation-rework.md @@ -0,0 +1,138 @@ +# PostgreSQL Adaptation Rework + +## Summary + +This rework does not continue the original PostgreSQL attempt on top of the +same synchronous database abstraction. It starts from `develop` and addresses +the architectural concerns that blocked the previous proposal: + +- remove the synchronous PostgreSQL driver plus thread-per-operation workaround +- replace the monolithic database trait with narrower persistence stores +- move SQL backends to a shared async `sqlx` substrate +- treat migrations as the schema source of truth +- widen completed/download counters to `u64` internally +- keep BitTorrent protocol compatibility by clamping only at the protocol edge + +The goal is to make PostgreSQL a normal backend, not a special case. + +## Main Changes + +### Persistence redesign + +- split the old `Database` trait into: + - `SchemaMigrator` + - `TorrentMetricsStore` + - `WhitelistStore` + - `AuthKeyStore` +- update the tracker core container and persisted repositories to depend on the + narrow store traits instead of a single all-purpose database object + +### Async SQL backend + +- replace the SQL drivers used by SQLite and MySQL with `sqlx` +- add a PostgreSQL backend implemented on the same async substrate +- remove the old synchronous PostgreSQL execution model that spawned threads to + avoid blocking the Tokio runtime + +### Schema and data model alignment + +- make migrations the source of truth for SQL schema +- add PostgreSQL migrations +- add a widening migration for download counters +- normalize `InfoHash` storage across SQL backends + +### BitTorrent protocol handling + +- widen internal completed/download counters from `u32` to `u64` +- keep HTTP responses compatible +- clamp UDP scrape counters explicitly at the protocol boundary instead of + relying on unchecked narrowing casts + +### Configuration and tooling + +- add PostgreSQL configuration support and a default PostgreSQL container config +- add compatibility and end-to-end QA scripts +- add a before/after benchmark harness + +## Benchmark Snapshot + +Benchmarks were run with release builds and the same black-box workload against +the old branch and this rework: + +- tracker startup to health ready +- HTTP announce lifecycle (`started -> completed`) +- REST whitelist add/add-concurrent/reload +- REST auth key add/add-concurrent/reload + +Environment: + +- SQLite `3.51.3` +- MySQL `8.0` +- PostgreSQL `16` + +### SQLite + +- startup: effectively unchanged +- announce path: small improvement, about `+2%` to `+4%` +- whitelist/auth-key writes: roughly flat +- reload paths: slower, around `0.64x` to `0.67x` of the old throughput + +### MySQL + +- startup: slightly faster +- announce path: about `+4%` to `+15%` +- sequential whitelist/auth-key writes: about `+16%` to `+19%` +- reload paths: mildly slower, around `0.88x` to `0.90x` + +### PostgreSQL + +- startup: slightly faster +- announce path: effectively neutral to slightly better +- whitelist writes: up to `+61%` +- auth-key writes: up to `+30%` +- auth-key reload: about `+38%` + +The main outcome is that PostgreSQL now performs competitively without relying +on the previously rejected synchronous thread-spawning execution model. + +## Draft Reply To Maintainer + +> I reworked this from `develop` instead of continuing the previous PostgreSQL +> branch on top of the same database abstraction. +> +> The main idea this time was to address the architectural concerns first, and +> then add PostgreSQL on top of the new persistence layer, rather than adding a +> PostgreSQL special case. +> +> Concretely, I made these changes: +> +> - replaced the monolithic database trait with narrower persistence stores for +> torrent metrics, whitelist, auth keys, and schema migration +> - moved the SQL backends to a shared async `sqlx` substrate +> - removed the synchronous PostgreSQL driver and the thread-per-operation +> workaround +> - made migrations the source of truth for schema and added PostgreSQL +> migrations +> - widened completed/download counters to `u64` internally and clamped only at +> the BitTorrent protocol boundary where 32-bit fields are still required +> - added PostgreSQL config support, integration coverage, and QA scripts +> +> I also ran a before/after comparison with the same black-box workloads against +> SQLite, MySQL, and PostgreSQL. +> +> High-level results: +> +> - SQLite is mostly neutral, except the reload paths are slower +> - MySQL is modestly better on announce and write-heavy paths, with slightly +> slower reloads +> - PostgreSQL shows the clearest improvement: startup is slightly better, +> announce performance is at least on par, and write-heavy persistence paths +> are noticeably faster than the previous attempt +> +> For PostgreSQL specifically, the point of this rework was not only to make it +> work, but to make it fit the project without the previously rejected blocking +> execution model. +> +> Does this overall direction look acceptable? If so, I can either keep it as a +> single PR or split it into smaller reviewable pieces, depending on how you +> would prefer to review it. diff --git a/packages/axum-health-check-api-server/tests/server/contract.rs b/packages/axum-health-check-api-server/tests/server/contract.rs index af1c0cff9..93d513177 100644 --- a/packages/axum-health-check-api-server/tests/server/contract.rs +++ b/packages/axum-health-check-api-server/tests/server/contract.rs @@ -326,7 +326,14 @@ mod udp { let details = report.details.first().expect("it should have some details"); assert_eq!(details.binding, binding); - assert_eq!(details.result, Err("Timed Out".to_string())); + assert!( + details + .result + .as_ref() + .is_err_and(|error| error.contains("Timed Out") || error.contains("Connection refused")), + "unexpected udp health-check error: {:?}", + details.result + ); assert_eq!(details.info, format!("checking the udp tracker health check at: {binding}")); env.stop().await.expect("it should stop the service"); diff --git a/packages/axum-http-tracker-server/src/v1/handlers/announce.rs b/packages/axum-http-tracker-server/src/v1/handlers/announce.rs index ce718cd30..a1d7019d7 100644 --- a/packages/axum-http-tracker-server/src/v1/handlers/announce.rs +++ b/packages/axum-http-tracker-server/src/v1/handlers/announce.rs @@ -160,7 +160,7 @@ mod tests { let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); let authentication_service = Arc::new(AuthenticationService::new(&config.core, &in_memory_key_repository)); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let announce_handler = Arc::new(AnnounceHandler::new( &config.core, &whitelist_authorization, diff --git a/packages/axum-rest-tracker-api-server/tests/server/mod.rs b/packages/axum-rest-tracker-api-server/tests/server/mod.rs index 9dea49a4c..1b3e9a759 100644 --- a/packages/axum-rest-tracker-api-server/tests/server/mod.rs +++ b/packages/axum-rest-tracker-api-server/tests/server/mod.rs @@ -1,9 +1,7 @@ pub mod connection_info; pub mod v1; -use std::sync::Arc; - -use bittorrent_tracker_core::databases::Database; +use bittorrent_tracker_core::databases::Persistence; /// It forces a database error by dropping all tables. That makes all queries /// fail. @@ -14,6 +12,7 @@ use bittorrent_tracker_core::databases::Database; /// /// - Inject a database mock in the future. /// - Inject directly the database reference passed to the Tracker type. -pub fn force_database_error(tracker: &Arc>) { - tracker.drop_database_tables().unwrap(); +pub async fn force_database_error(tracker: &Persistence) { + tracker.schema_migrator().create_database_tables().await.unwrap(); + tracker.schema_migrator().drop_database_tables().await.unwrap(); } diff --git a/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/auth_key.rs b/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/auth_key.rs index 3781f4f60..db491bd1d 100644 --- a/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/auth_key.rs +++ b/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/auth_key.rs @@ -135,7 +135,7 @@ async fn should_fail_when_the_auth_key_cannot_be_generated() { let env = Started::new(&configuration::ephemeral().into()).await; - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let request_id = Uuid::new_v4(); @@ -315,7 +315,7 @@ async fn should_fail_when_the_auth_key_cannot_be_deleted() { .await .unwrap(); - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let request_id = Uuid::new_v4(); @@ -433,7 +433,7 @@ async fn should_fail_when_keys_cannot_be_reloaded() { .await .unwrap(); - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let response = Client::new(env.get_connection_info()) .unwrap() @@ -598,7 +598,7 @@ mod deprecated_generate_key_endpoint { let env = Started::new(&configuration::ephemeral().into()).await; - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let request_id = Uuid::new_v4(); let seconds_valid = 60; diff --git a/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/whitelist.rs b/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/whitelist.rs index 61fc233d0..558c3584f 100644 --- a/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/whitelist.rs +++ b/packages/axum-rest-tracker-api-server/tests/server/v1/contract/context/whitelist.rs @@ -115,7 +115,7 @@ async fn should_fail_when_the_torrent_cannot_be_whitelisted() { let info_hash = "9e0217d0fa71c87332cd8bf9dbeabcb2c2cf3c4d".to_owned(); // DevSkim: ignore DS173237 - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let request_id = Uuid::new_v4(); @@ -266,7 +266,7 @@ async fn should_fail_when_the_torrent_cannot_be_removed_from_the_whitelist() { .await .unwrap(); - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let request_id = Uuid::new_v4(); @@ -392,7 +392,7 @@ async fn should_fail_when_the_whitelist_cannot_be_reloaded_from_the_database() { .await .unwrap(); - force_database_error(&env.container.tracker_core_container.database); + force_database_error(&env.container.tracker_core_container.database).await; let request_id = Uuid::new_v4(); diff --git a/packages/configuration/src/v2_0_0/database.rs b/packages/configuration/src/v2_0_0/database.rs index c2b24d809..f4c2259f7 100644 --- a/packages/configuration/src/v2_0_0/database.rs +++ b/packages/configuration/src/v2_0_0/database.rs @@ -5,15 +5,17 @@ use url::Url; #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] pub struct Database { // Database configuration - /// Database driver. Possible values are: `sqlite3`, and `mysql`. + /// Database driver. Possible values are: `sqlite3`, `mysql`, and `postgresql`. #[serde(default = "Database::default_driver")] pub driver: Driver, /// Database connection string. The format depends on the database driver. /// For `sqlite3`, the format is `path/to/database.db`, for example: /// `./storage/tracker/lib/database/sqlite3.db`. - /// For `Mysql`, the format is `mysql://db_user:db_user_password:port/db_name`, for + /// For `Mysql`, the format is `mysql://db_user:db_user_password@host:port/db_name`, for /// example: `mysql://root:password@localhost:3306/torrust`. + /// For `PostgreSQL`, the format is `postgresql://db_user:db_user_password@host:port/db_name`, + /// for example: `postgresql://postgres:password@localhost:5432/torrust`. #[serde(default = "Database::default_path")] pub path: String, } @@ -46,8 +48,8 @@ impl Database { Driver::Sqlite3 => { // Nothing to mask } - Driver::MySQL => { - let mut url = Url::parse(&self.path).expect("path for MySQL driver should be a valid URL"); + Driver::MySQL | Driver::PostgreSQL => { + let mut url = Url::parse(&self.path).expect("path for SQL URL-based drivers should be a valid URL"); url.set_password(Some("***")).expect("url password should be changed"); self.path = url.to_string(); } @@ -63,6 +65,8 @@ pub enum Driver { Sqlite3, /// The `MySQL` database driver. MySQL, + /// The `PostgreSQL` database driver. + PostgreSQL, } #[cfg(test)] @@ -81,4 +85,16 @@ mod tests { assert_eq!(database.path, "mysql://root:***@localhost:3306/torrust".to_string()); } + + #[test] + fn it_should_allow_masking_the_postgresql_user_password() { + let mut database = Database { + driver: Driver::PostgreSQL, + path: "postgresql://postgres:password@localhost:5432/torrust".to_string(), + }; + + database.mask_secrets(); + + assert_eq!(database.path, "postgresql://postgres:***@localhost:5432/torrust".to_string()); + } } diff --git a/packages/http-protocol/src/v1/responses/scrape.rs b/packages/http-protocol/src/v1/responses/scrape.rs index 022735abc..bcf0811f5 100644 --- a/packages/http-protocol/src/v1/responses/scrape.rs +++ b/packages/http-protocol/src/v1/responses/scrape.rs @@ -5,6 +5,7 @@ use std::borrow::Cow; use torrust_tracker_contrib_bencode::{ben_int, ben_map, BMutAccess}; use torrust_tracker_primitives::core::ScrapeData; +use torrust_tracker_primitives::NumberOfDownloads; /// The `Scrape` response for the HTTP tracker. /// @@ -60,7 +61,7 @@ impl Bencoded { Cow::from(info_hash.bytes().to_vec()), ben_map! { "complete" => ben_int!(i64::from(value.complete)), - "downloaded" => ben_int!(i64::from(value.downloaded)), + "downloaded" => ben_int!(bencode_download_count(value.downloaded)), "incomplete" => ben_int!(i64::from(value.incomplete)) }, ); @@ -73,6 +74,10 @@ impl Bencoded { } } +fn bencode_download_count(downloaded: NumberOfDownloads) -> i64 { + i64::try_from(downloaded).unwrap_or(i64::MAX) +} + impl From for Bencoded { fn from(scrape_data: ScrapeData) -> Self { Self { scrape_data } @@ -131,5 +136,25 @@ mod tests { String::from_utf8(expected_bytes.to_vec()).unwrap() ); } + + #[test] + fn should_saturate_large_download_counts() { + let info_hash = InfoHash::from_bytes(&[0x69; 20]); + let mut scrape_data = ScrapeData::empty(); + scrape_data.add_file( + &info_hash, + SwarmMetadata { + complete: 1, + downloaded: u64::MAX, + incomplete: 3, + }, + ); + + let response = Bencoded::from(scrape_data); + let bytes = response.body(); + let body = String::from_utf8(bytes).unwrap(); + + assert!(body.contains("downloadedi9223372036854775807e")); + } } } diff --git a/packages/http-tracker-core/benches/helpers/util.rs b/packages/http-tracker-core/benches/helpers/util.rs index 028d7c535..78079a9b7 100644 --- a/packages/http-tracker-core/benches/helpers/util.rs +++ b/packages/http-tracker-core/benches/helpers/util.rs @@ -48,7 +48,7 @@ pub fn initialize_core_tracker_services_with_config(config: &Configuration) -> ( let core_config = Arc::new(config.core.clone()); let database = initialize_database(&config.core); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_authorization = Arc::new(WhitelistAuthorization::new(&config.core, &in_memory_whitelist.clone())); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); diff --git a/packages/http-tracker-core/src/services/announce.rs b/packages/http-tracker-core/src/services/announce.rs index 766f08c12..6a27b1d0b 100644 --- a/packages/http-tracker-core/src/services/announce.rs +++ b/packages/http-tracker-core/src/services/announce.rs @@ -242,7 +242,7 @@ mod tests { let core_config = Arc::new(config.core.clone()); let database = initialize_database(&config.core); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_authorization = Arc::new(WhitelistAuthorization::new(&config.core, &in_memory_whitelist.clone())); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); diff --git a/packages/http-tracker-core/src/services/scrape.rs b/packages/http-tracker-core/src/services/scrape.rs index 4587bc90a..6e7e4c951 100644 --- a/packages/http-tracker-core/src/services/scrape.rs +++ b/packages/http-tracker-core/src/services/scrape.rs @@ -200,7 +200,7 @@ mod tests { let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_authorization = Arc::new(WhitelistAuthorization::new(&config.core, &in_memory_whitelist.clone())); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); let authentication_service = Arc::new(AuthenticationService::new(&config.core, &in_memory_key_repository)); diff --git a/packages/primitives/src/lib.rs b/packages/primitives/src/lib.rs index ec2edda97..fe8c20a47 100644 --- a/packages/primitives/src/lib.rs +++ b/packages/primitives/src/lib.rs @@ -18,5 +18,5 @@ use bittorrent_primitives::info_hash::InfoHash; /// Duration since the Unix Epoch. pub type DurationSinceUnixEpoch = Duration; -pub type NumberOfDownloads = u32; +pub type NumberOfDownloads = u64; pub type NumberOfDownloadsBTreeMap = BTreeMap; diff --git a/packages/primitives/src/swarm_metadata.rs b/packages/primitives/src/swarm_metadata.rs index 57ba816d3..1bfea0b53 100644 --- a/packages/primitives/src/swarm_metadata.rs +++ b/packages/primitives/src/swarm_metadata.rs @@ -2,6 +2,8 @@ use std::ops::AddAssign; use derive_more::Constructor; +use crate::NumberOfDownloads; + /// Swarm statistics for one torrent. /// /// Swarm metadata dictionary in the scrape response. @@ -11,14 +13,23 @@ use derive_more::Constructor; pub struct SwarmMetadata { /// (i.e `completed`): The number of peers that have ever completed /// downloading a given torrent. - pub downloaded: u32, + /// + /// This uses `u64` because it is persisted as a cumulative counter and can + /// grow independently from the active peer counts below. + pub downloaded: NumberOfDownloads, /// (i.e `seeders`): The number of active peers that have completed /// downloading (seeders) a given torrent. + /// + /// Active peer counts stay bounded by the swarm population, so `u32` + /// remains sufficient here. pub complete: u32, /// (i.e `leechers`): The number of active peers that have not completed /// downloading (leechers) a given torrent. + /// + /// Active peer counts stay bounded by the swarm population, so `u32` + /// remains sufficient here. pub incomplete: u32, } @@ -29,7 +40,7 @@ impl SwarmMetadata { } #[must_use] - pub fn downloads(&self) -> u32 { + pub fn downloads(&self) -> NumberOfDownloads { self.downloaded } diff --git a/packages/swarm-coordination-registry/src/swarm/coordinator.rs b/packages/swarm-coordination-registry/src/swarm/coordinator.rs index 433ab9d32..ea66eb236 100644 --- a/packages/swarm-coordination-registry/src/swarm/coordinator.rs +++ b/packages/swarm-coordination-registry/src/swarm/coordinator.rs @@ -9,7 +9,7 @@ use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::peer::{self, Peer, PeerAnnouncement}; use torrust_tracker_primitives::swarm_metadata::SwarmMetadata; -use torrust_tracker_primitives::DurationSinceUnixEpoch; +use torrust_tracker_primitives::{DurationSinceUnixEpoch, NumberOfDownloads}; use crate::event::sender::Sender; use crate::event::Event; @@ -24,7 +24,7 @@ pub struct Coordinator { impl Coordinator { #[must_use] - pub fn new(info_hash: &InfoHash, downloaded: u32, event_sender: Sender) -> Self { + pub fn new(info_hash: &InfoHash, downloaded: NumberOfDownloads, event_sender: Sender) -> Self { Self { info_hash: *info_hash, peers: BTreeMap::new(), diff --git a/packages/swarm-coordination-registry/src/swarm/registry.rs b/packages/swarm-coordination-registry/src/swarm/registry.rs index c8e98f307..26f622e86 100644 --- a/packages/swarm-coordination-registry/src/swarm/registry.rs +++ b/packages/swarm-coordination-registry/src/swarm/registry.rs @@ -403,7 +403,7 @@ impl Registry { let stats = swarm.metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/entry/mod.rs b/packages/torrent-repository-benchmarking/src/entry/mod.rs index b920839d9..b325b0e8b 100644 --- a/packages/torrent-repository-benchmarking/src/entry/mod.rs +++ b/packages/torrent-repository-benchmarking/src/entry/mod.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::swarm_metadata::SwarmMetadata; -use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch}; +use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads}; use self::peer_list::PeerList; @@ -88,5 +88,5 @@ pub struct Torrent { /// A network of peers that are all trying to download the torrent associated to this entry pub(crate) swarm: PeerList, /// The number of peers that have ever completed downloading the torrent associated to this entry - pub(crate) downloaded: u32, + pub(crate) downloaded: NumberOfDownloads, } diff --git a/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs index fec94b4a5..7c03f6b06 100644 --- a/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs @@ -52,7 +52,7 @@ where for entry in &self.torrents { let stats = entry.value().lock().expect("it should get a lock").get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs index 5000579dd..d6324ace3 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs @@ -70,7 +70,7 @@ where for entry in self.get_torrents().values() { let stats = entry.get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs index 085256ff1..9cfeca9ae 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs @@ -65,7 +65,7 @@ where for entry in self.get_torrents().values() { let stats = entry.lock().expect("it should get a lock").get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs index 9fd451149..1f0f806f7 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs @@ -93,7 +93,7 @@ where for entry in entries { let stats = entry.lock().await.get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs index e85200aeb..bdd67e9c2 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs @@ -90,7 +90,7 @@ where for entry in self.get_torrents().await.values() { let stats = entry.get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs index 8d6584713..e61bb9ea3 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs @@ -84,7 +84,7 @@ where for entry in self.get_torrents().await.values() { let stats = entry.get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs index c8f499e03..8cf23b843 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs @@ -87,7 +87,7 @@ where for entry in self.get_torrents().await.values() { let stats = entry.get_swarm_metadata().await; metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs index 0432b13d0..abbe3111d 100644 --- a/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs @@ -75,7 +75,7 @@ where for entry in &self.torrents { let stats = entry.value().lock().expect("it should get a lock").get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } @@ -168,7 +168,7 @@ where for entry in &self.torrents { let stats = entry.value().read().get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } @@ -261,7 +261,7 @@ where for entry in &self.torrents { let stats = entry.value().lock().get_swarm_metadata(); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_torrents += 1; } diff --git a/packages/torrent-repository-benchmarking/tests/repository/mod.rs b/packages/torrent-repository-benchmarking/tests/repository/mod.rs index fb0b8fcff..461b13d74 100644 --- a/packages/torrent-repository-benchmarking/tests/repository/mod.rs +++ b/packages/torrent-repository-benchmarking/tests/repository/mod.rs @@ -176,7 +176,7 @@ fn persistent_single() -> NumberOfDownloadsBTreeMap { let hash = &mut DefaultHasher::default(); hash.write_u8(1); - let t = [(InfoHash::from(&hash.clone()), 0_u32)]; + let t = [(InfoHash::from(&hash.clone()), 0_u64)]; t.iter().copied().collect() } @@ -192,7 +192,7 @@ fn persistent_three() -> NumberOfDownloadsBTreeMap { hash.write_u8(3); let info_3 = InfoHash::from(&hash.clone()); - let t = [(info_1, 1_u32), (info_2, 2_u32), (info_3, 3_u32)]; + let t = [(info_1, 1_u64), (info_2, 2_u64), (info_3, 3_u64)]; t.iter().copied().collect() } @@ -412,7 +412,7 @@ async fn it_should_get_metrics( metrics.total_torrents += 1; metrics.total_incomplete += u64::from(stats.incomplete); metrics.total_complete += u64::from(stats.complete); - metrics.total_downloaded += u64::from(stats.downloaded); + metrics.total_downloaded += stats.downloaded; } assert_eq!(repo.get_metrics().await, metrics); diff --git a/packages/tracker-client/src/udp/client.rs b/packages/tracker-client/src/udp/client.rs index 94c882d29..7f07b5a15 100644 --- a/packages/tracker-client/src/udp/client.rs +++ b/packages/tracker-client/src/udp/client.rs @@ -264,7 +264,10 @@ pub async fn check(service_binding: &ServiceBinding) -> Result { Err("Timed Out".to_string()) } response = client.receive() => { - process(response.unwrap()) + match response { + Ok(response) => process(response), + Err(err) => Err(err.to_string()), + } } } } diff --git a/packages/tracker-core/Cargo.toml b/packages/tracker-core/Cargo.toml index fb864cde7..5d4055644 100644 --- a/packages/tracker-core/Cargo.toml +++ b/packages/tracker-core/Cargo.toml @@ -15,16 +15,15 @@ version.workspace = true [dependencies] aquatic_udp_protocol = "0" +async-trait = "0" bittorrent-primitives = "0.1.0" chrono = { version = "0", default-features = false, features = [ "clock" ] } derive_more = { version = "2", features = [ "as_ref", "constructor", "from" ] } mockall = "0" -r2d2 = "0" -r2d2_mysql = "25" -r2d2_sqlite = { version = "0", features = [ "bundled" ] } rand = "0" serde = { version = "1", features = [ "derive" ] } serde_json = { version = "1", features = [ "preserve_order" ] } +sqlx = { version = "0.8", features = [ "runtime-tokio-rustls", "sqlite", "mysql", "postgres", "migrate" ] } thiserror = "2" tokio = { version = "1", features = [ "macros", "net", "rt-multi-thread", "signal", "sync" ] } tokio-util = "0.7.15" diff --git a/packages/tracker-core/migrations/README.md b/packages/tracker-core/migrations/README.md index 090c46ccb..d4f8637bc 100644 --- a/packages/tracker-core/migrations/README.md +++ b/packages/tracker-core/migrations/README.md @@ -1,5 +1,15 @@ # Database Migrations -We don't support automatic migrations yet. The tracker creates all the needed tables when it starts. The SQL sentences are hardcoded in each database driver. +The tracker uses SQL migrations as the schema source of truth for all SQL +backends: -The migrations in this folder were introduced to add some new changes (permanent keys) and to allow users to migrate to the new version. In the future, we will remove the hardcoded SQL and start using a Rust crate for database migrations. For the time being, if you are using the initial schema described in the migration `20240730183000_torrust_tracker_create_all_tables.sql` you will need to run all the subsequent migrations manually. +- `mysql/` +- `postgresql/` +- `sqlite/` + +The tracker applies these migrations automatically when a database-backed store +is first used. + +The files intentionally remain split per backend because SQL syntax and column +types differ across engines. Migration ordering is shared by timestamp prefix so +the schema evolution remains aligned across backends. diff --git a/packages/tracker-core/migrations/mysql/20240730183000_torrust_tracker_create_all_tables.sql b/packages/tracker-core/migrations/mysql/20240730183000_torrust_tracker_create_all_tables.sql index ab160bd75..665e2b659 100644 --- a/packages/tracker-core/migrations/mysql/20240730183000_torrust_tracker_create_all_tables.sql +++ b/packages/tracker-core/migrations/mysql/20240730183000_torrust_tracker_create_all_tables.sql @@ -4,7 +4,7 @@ CREATE TABLE info_hash VARCHAR(40) NOT NULL UNIQUE ); -# todo: rename to `torrent_metrics` +-- todo: rename to `torrent_metrics` CREATE TABLE IF NOT EXISTS torrents ( id integer PRIMARY KEY AUTO_INCREMENT, @@ -19,4 +19,4 @@ CREATE TABLE `valid_until` INT (10) NOT NULL, PRIMARY KEY (`id`), UNIQUE (`key`) - ); \ No newline at end of file + ); diff --git a/packages/tracker-core/migrations/mysql/20260409120000_torrust_tracker_widen_download_counters.sql b/packages/tracker-core/migrations/mysql/20260409120000_torrust_tracker_widen_download_counters.sql new file mode 100644 index 000000000..b9adb3a27 --- /dev/null +++ b/packages/tracker-core/migrations/mysql/20260409120000_torrust_tracker_widen_download_counters.sql @@ -0,0 +1,5 @@ +ALTER TABLE torrents + MODIFY completed BIGINT NOT NULL DEFAULT 0; + +ALTER TABLE torrent_aggregate_metrics + MODIFY value BIGINT NOT NULL DEFAULT 0; diff --git a/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql b/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql new file mode 100644 index 000000000..05b8de4ed --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql @@ -0,0 +1,19 @@ +CREATE TABLE + IF NOT EXISTS whitelist ( + id SERIAL PRIMARY KEY, + info_hash VARCHAR(40) NOT NULL UNIQUE + ); + +CREATE TABLE + IF NOT EXISTS torrents ( + id SERIAL PRIMARY KEY, + info_hash VARCHAR(40) NOT NULL UNIQUE, + completed INTEGER DEFAULT 0 NOT NULL + ); + +CREATE TABLE + IF NOT EXISTS keys ( + id SERIAL PRIMARY KEY, + key VARCHAR(32) NOT NULL UNIQUE, + valid_until BIGINT NOT NULL + ); diff --git a/packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql b/packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql new file mode 100644 index 000000000..c7357d63f --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql @@ -0,0 +1,2 @@ +ALTER TABLE keys + ALTER COLUMN valid_until DROP NOT NULL; diff --git a/packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql b/packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql new file mode 100644 index 000000000..1a41f31c0 --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql @@ -0,0 +1,6 @@ +CREATE TABLE + IF NOT EXISTS torrent_aggregate_metrics ( + id SERIAL PRIMARY KEY, + metric_name VARCHAR(50) NOT NULL UNIQUE, + value INTEGER DEFAULT 0 NOT NULL + ); diff --git a/packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql b/packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql new file mode 100644 index 000000000..3fdf9fcb2 --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql @@ -0,0 +1,9 @@ +ALTER TABLE torrents + ALTER COLUMN completed TYPE BIGINT, + ALTER COLUMN completed SET DEFAULT 0, + ALTER COLUMN completed SET NOT NULL; + +ALTER TABLE torrent_aggregate_metrics + ALTER COLUMN value TYPE BIGINT, + ALTER COLUMN value SET DEFAULT 0, + ALTER COLUMN value SET NOT NULL; diff --git a/packages/tracker-core/migrations/sqlite/20240730183000_torrust_tracker_create_all_tables.sql b/packages/tracker-core/migrations/sqlite/20240730183000_torrust_tracker_create_all_tables.sql index c5bcad926..0a9f2cf69 100644 --- a/packages/tracker-core/migrations/sqlite/20240730183000_torrust_tracker_create_all_tables.sql +++ b/packages/tracker-core/migrations/sqlite/20240730183000_torrust_tracker_create_all_tables.sql @@ -4,7 +4,7 @@ CREATE TABLE info_hash TEXT NOT NULL UNIQUE ); -# todo: rename to `torrent_metrics` +-- todo: rename to `torrent_metrics` CREATE TABLE IF NOT EXISTS torrents ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -17,4 +17,4 @@ CREATE TABLE id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL UNIQUE, valid_until INTEGER NOT NULL - ); \ No newline at end of file + ); diff --git a/packages/tracker-core/migrations/sqlite/20260409120000_torrust_tracker_widen_download_counters.sql b/packages/tracker-core/migrations/sqlite/20260409120000_torrust_tracker_widen_download_counters.sql new file mode 100644 index 000000000..ce63c4aa3 --- /dev/null +++ b/packages/tracker-core/migrations/sqlite/20260409120000_torrust_tracker_widen_download_counters.sql @@ -0,0 +1,3 @@ +-- SQLite stores INTEGER values as signed 64-bit integers already. +-- This migration is intentionally a no-op so the migration history stays +-- aligned with the MySQL and PostgreSQL backends. diff --git a/packages/tracker-core/src/announce_handler.rs b/packages/tracker-core/src/announce_handler.rs index 0b6bffd31..9e7eabbdb 100644 --- a/packages/tracker-core/src/announce_handler.rs +++ b/packages/tracker-core/src/announce_handler.rs @@ -167,20 +167,20 @@ impl AnnounceHandler { peer.change_ip(&assign_ip_address_to_peer(remote_client_ip, self.config.net.external_ip)); self.in_memory_torrent_repository - .handle_announcement(info_hash, peer, self.load_downloads_metric_if_needed(info_hash)?) + .handle_announcement(info_hash, peer, self.load_downloads_metric_if_needed(info_hash).await?) .await; Ok(self.build_announce_data(info_hash, peer, peers_wanted).await) } /// Loads the number of downloads for a torrent if needed. - fn load_downloads_metric_if_needed( + async fn load_downloads_metric_if_needed( &self, info_hash: &InfoHash, ) -> Result, databases::error::Error> { if self.config.tracker_policy.persistent_torrent_completed_stat && !self.in_memory_torrent_repository.contains(info_hash) { - Ok(self.db_downloads_metric_repository.load_torrent_downloads(info_hash)?) + Ok(self.db_downloads_metric_repository.load_torrent_downloads(info_hash).await?) } else { Ok(None) } diff --git a/packages/tracker-core/src/authentication/handler.rs b/packages/tracker-core/src/authentication/handler.rs index 178895b8d..d7a9e0a58 100644 --- a/packages/tracker-core/src/authentication/handler.rs +++ b/packages/tracker-core/src/authentication/handler.rs @@ -182,7 +182,7 @@ impl KeysHandler { pub async fn generate_expiring_peer_key(&self, lifetime: Option) -> Result { let peer_key = key::generate_key(lifetime); - self.db_key_repository.add(&peer_key)?; + self.db_key_repository.add(&peer_key).await?; self.in_memory_key_repository.insert(&peer_key).await; @@ -226,10 +226,7 @@ impl KeysHandler { ) -> Result { let peer_key = PeerKey { key, valid_until }; - // code-review: should we return a friendly error instead of the DB - // constrain error when the key already exist? For now, it's returning - // the specif error for each DB driver when a UNIQUE constrain fails. - self.db_key_repository.add(&peer_key)?; + self.db_key_repository.add(&peer_key).await?; self.in_memory_key_repository.insert(&peer_key).await; @@ -249,7 +246,7 @@ impl KeysHandler { /// Returns a `databases::error::Error` if the key cannot be removed from /// the database. pub async fn remove_peer_key(&self, key: &Key) -> Result<(), databases::error::Error> { - self.db_key_repository.remove(key)?; + self.db_key_repository.remove(key).await?; self.remove_in_memory_auth_key(key).await; @@ -277,7 +274,7 @@ impl KeysHandler { /// /// Returns a `databases::error::Error` if there is an issue loading the keys from the database. pub async fn load_peer_keys_from_database(&self) -> Result<(), databases::error::Error> { - let keys_from_database = self.db_key_repository.load_keys()?; + let keys_from_database = self.db_key_repository.load_keys().await?; self.in_memory_key_repository.reset_with(keys_from_database).await; @@ -299,7 +296,7 @@ mod tests { use crate::authentication::key::repository::in_memory::InMemoryKeyRepository; use crate::authentication::key::repository::persisted::DatabaseKeyRepository; use crate::databases::setup::initialize_database; - use crate::databases::Database; + use crate::databases::AuthKeyStore; fn instantiate_keys_handler() -> KeysHandler { let config = configuration::ephemeral_private(); @@ -307,7 +304,7 @@ mod tests { instantiate_keys_handler_with_configuration(&config) } - fn instantiate_keys_handler_with_database(database: &Arc>) -> KeysHandler { + fn instantiate_keys_handler_with_database(database: Arc) -> KeysHandler { let db_key_repository = Arc::new(DatabaseKeyRepository::new(database)); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); @@ -318,7 +315,7 @@ mod tests { // todo: pass only Core configuration let database = initialize_database(&config.core); - let db_key_repository = Arc::new(DatabaseKeyRepository::new(&database)); + let db_key_repository = Arc::new(DatabaseKeyRepository::new(database.auth_key_store())); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); KeysHandler::new(&db_key_repository, &in_memory_key_repository) @@ -363,7 +360,7 @@ mod tests { use crate::authentication::handler::AddKeyRequest; use crate::authentication::PeerKey; use crate::databases::driver::Driver; - use crate::databases::{self, Database, MockDatabase}; + use crate::databases::{self, AuthKeyStore, MockAuthKeyStore}; use crate::error::PeerKeyError; use crate::CurrentClock; @@ -392,7 +389,7 @@ mod tests { // The key should be valid the next 60 seconds. let expected_valid_until = clock::Stopped::now_add(&Duration::from_secs(60)).unwrap(); - let mut database_mock = MockDatabase::default(); + let mut database_mock = MockAuthKeyStore::default(); database_mock .expect_add_key_to_keys() .with(function(move |peer_key: &PeerKey| { @@ -405,9 +402,9 @@ mod tests { driver: Driver::Sqlite3, }) }); - let database_mock: Arc> = Arc::new(Box::new(database_mock)); + let database_mock: Arc = Arc::new(database_mock); - let keys_handler = instantiate_keys_handler_with_database(&database_mock); + let keys_handler = instantiate_keys_handler_with_database(database_mock); let result = keys_handler .add_peer_key(AddKeyRequest { @@ -435,7 +432,7 @@ mod tests { use crate::authentication::handler::AddKeyRequest; use crate::authentication::{Key, PeerKey}; use crate::databases::driver::Driver; - use crate::databases::{self, Database, MockDatabase}; + use crate::databases::{self, AuthKeyStore, MockAuthKeyStore}; use crate::error::PeerKeyError; use crate::CurrentClock; @@ -499,7 +496,7 @@ mod tests { valid_until: Some(expected_valid_until), }; - let mut database_mock = MockDatabase::default(); + let mut database_mock = MockAuthKeyStore::default(); database_mock .expect_add_key_to_keys() .with(predicate::eq(expected_peer_key)) @@ -510,9 +507,9 @@ mod tests { driver: Driver::Sqlite3, }) }); - let database_mock: Arc> = Arc::new(Box::new(database_mock)); + let database_mock: Arc = Arc::new(database_mock); - let keys_handler = instantiate_keys_handler_with_database(&database_mock); + let keys_handler = instantiate_keys_handler_with_database(database_mock); let result = keys_handler .add_peer_key(AddKeyRequest { @@ -541,7 +538,7 @@ mod tests { use crate::authentication::handler::AddKeyRequest; use crate::authentication::PeerKey; use crate::databases::driver::Driver; - use crate::databases::{self, Database, MockDatabase}; + use crate::databases::{self, AuthKeyStore, MockAuthKeyStore}; use crate::error::PeerKeyError; #[tokio::test] @@ -570,7 +567,7 @@ mod tests { #[tokio::test] async fn it_should_fail_adding_a_randomly_generated_key_when_there_is_a_database_error() { - let mut database_mock = MockDatabase::default(); + let mut database_mock = MockAuthKeyStore::default(); database_mock .expect_add_key_to_keys() .with(function(move |peer_key: &PeerKey| peer_key.valid_until.is_none())) @@ -581,9 +578,9 @@ mod tests { driver: Driver::Sqlite3, }) }); - let database_mock: Arc> = Arc::new(Box::new(database_mock)); + let database_mock: Arc = Arc::new(database_mock); - let keys_handler = instantiate_keys_handler_with_database(&database_mock); + let keys_handler = instantiate_keys_handler_with_database(database_mock); let result = keys_handler .add_peer_key(AddKeyRequest { @@ -609,7 +606,7 @@ mod tests { use crate::authentication::handler::AddKeyRequest; use crate::authentication::{Key, PeerKey}; use crate::databases::driver::Driver; - use crate::databases::{self, Database, MockDatabase}; + use crate::databases::{self, AuthKeyStore, MockAuthKeyStore}; use crate::error::PeerKeyError; #[tokio::test] @@ -654,7 +651,7 @@ mod tests { valid_until: None, }; - let mut database_mock = MockDatabase::default(); + let mut database_mock = MockAuthKeyStore::default(); database_mock .expect_add_key_to_keys() .with(predicate::eq(expected_peer_key)) @@ -665,9 +662,9 @@ mod tests { driver: Driver::Sqlite3, }) }); - let database_mock: Arc> = Arc::new(Box::new(database_mock)); + let database_mock: Arc = Arc::new(database_mock); - let keys_handler = instantiate_keys_handler_with_database(&database_mock); + let keys_handler = instantiate_keys_handler_with_database(database_mock); let result = keys_handler .add_peer_key(AddKeyRequest { diff --git a/packages/tracker-core/src/authentication/key/mod.rs b/packages/tracker-core/src/authentication/key/mod.rs index 44bbd0688..e86b1fc86 100644 --- a/packages/tracker-core/src/authentication/key/mod.rs +++ b/packages/tracker-core/src/authentication/key/mod.rs @@ -191,8 +191,8 @@ pub enum Error { MissingAuthKey { location: &'static Location<'static> }, } -impl From for Error { - fn from(e: r2d2_sqlite::rusqlite::Error) -> Self { +impl From for Error { + fn from(e: crate::databases::error::Error) -> Self { Error::KeyVerificationError { source: (Arc::new(e) as DynError).into(), } @@ -293,10 +293,14 @@ mod tests { mod the_key_verification_error { use crate::authentication::key; + use crate::databases::driver::Driver; #[test] fn could_be_a_database_error() { - let err = r2d2_sqlite::rusqlite::Error::InvalidQuery; + let err = crate::databases::error::Error::InsertFailed { + location: std::panic::Location::caller(), + driver: Driver::Sqlite3, + }; let err: key::Error = err.into(); diff --git a/packages/tracker-core/src/authentication/key/repository/persisted.rs b/packages/tracker-core/src/authentication/key/repository/persisted.rs index e84a23c9b..3bebfc995 100644 --- a/packages/tracker-core/src/authentication/key/repository/persisted.rs +++ b/packages/tracker-core/src/authentication/key/repository/persisted.rs @@ -2,15 +2,15 @@ use std::sync::Arc; use crate::authentication::key::{Key, PeerKey}; -use crate::databases::{self, Database}; +use crate::databases::{self, AuthKeyStore}; /// A repository for storing authentication keys in a persistent database. /// /// This repository provides methods to add, remove, and load authentication /// keys from the underlying database. It wraps an instance of a type -/// implementing the [`Database`] trait. +/// implementing the [`AuthKeyStore`] trait. pub struct DatabaseKeyRepository { - database: Arc>, + database: Arc, } impl DatabaseKeyRepository { @@ -18,16 +18,14 @@ impl DatabaseKeyRepository { /// /// # Arguments /// - /// * `database` - A shared reference to a boxed database implementation. + /// * `database` - A shared reference to an authentication key store. /// /// # Returns /// /// A new instance of `DatabaseKeyRepository` #[must_use] - pub fn new(database: &Arc>) -> Self { - Self { - database: database.clone(), - } + pub fn new(database: Arc) -> Self { + Self { database } } /// Adds a new authentication key to the database. @@ -39,8 +37,8 @@ impl DatabaseKeyRepository { /// # Errors /// /// Returns a [`databases::error::Error`] if the key cannot be added. - pub(crate) fn add(&self, peer_key: &PeerKey) -> Result<(), databases::error::Error> { - self.database.add_key_to_keys(peer_key)?; + pub(crate) async fn add(&self, peer_key: &PeerKey) -> Result<(), databases::error::Error> { + self.database.add_key_to_keys(peer_key).await?; Ok(()) } @@ -53,8 +51,8 @@ impl DatabaseKeyRepository { /// # Errors /// /// Returns a [`databases::error::Error`] if the key cannot be removed. - pub(crate) fn remove(&self, key: &Key) -> Result<(), databases::error::Error> { - self.database.remove_key_from_keys(key)?; + pub(crate) async fn remove(&self, key: &Key) -> Result<(), databases::error::Error> { + self.database.remove_key_from_keys(key).await?; Ok(()) } @@ -67,8 +65,8 @@ impl DatabaseKeyRepository { /// # Returns /// /// A vector containing all persisted [`PeerKey`] entries. - pub(crate) fn load_keys(&self) -> Result, databases::error::Error> { - let keys = self.database.load_keys()?; + pub(crate) async fn load_keys(&self) -> Result, databases::error::Error> { + let keys = self.database.load_keys().await?; Ok(keys) } } @@ -94,64 +92,64 @@ mod tests { config } - #[test] - fn persist_a_new_peer_key() { + #[tokio::test] + async fn persist_a_new_peer_key() { let configuration = ephemeral_configuration(); let database = initialize_database(&configuration); - let repository = DatabaseKeyRepository::new(&database); + let repository = DatabaseKeyRepository::new(database.auth_key_store()); let peer_key = PeerKey { key: Key::new("YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ").unwrap(), valid_until: Some(Duration::new(9999, 0)), }; - let result = repository.add(&peer_key); + let result = repository.add(&peer_key).await; assert!(result.is_ok()); - let keys = repository.load_keys().unwrap(); + let keys = repository.load_keys().await.unwrap(); assert_eq!(keys, vec!(peer_key)); } - #[test] - fn remove_a_persisted_peer_key() { + #[tokio::test] + async fn remove_a_persisted_peer_key() { let configuration = ephemeral_configuration(); let database = initialize_database(&configuration); - let repository = DatabaseKeyRepository::new(&database); + let repository = DatabaseKeyRepository::new(database.auth_key_store()); let peer_key = PeerKey { key: Key::new("YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ").unwrap(), valid_until: Some(Duration::new(9999, 0)), }; - let _unused = repository.add(&peer_key); + let _unused = repository.add(&peer_key).await; - let result = repository.remove(&peer_key.key); + let result = repository.remove(&peer_key.key).await; assert!(result.is_ok()); - let keys = repository.load_keys().unwrap(); + let keys = repository.load_keys().await.unwrap(); assert!(keys.is_empty()); } - #[test] - fn load_all_persisted_peer_keys() { + #[tokio::test] + async fn load_all_persisted_peer_keys() { let configuration = ephemeral_configuration(); let database = initialize_database(&configuration); - let repository = DatabaseKeyRepository::new(&database); + let repository = DatabaseKeyRepository::new(database.auth_key_store()); let peer_key = PeerKey { key: Key::new("YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ").unwrap(), valid_until: Some(Duration::new(9999, 0)), }; - let _unused = repository.add(&peer_key); + let _unused = repository.add(&peer_key).await; - let keys = repository.load_keys().unwrap(); + let keys = repository.load_keys().await.unwrap(); assert_eq!(keys, vec!(peer_key)); } diff --git a/packages/tracker-core/src/authentication/mod.rs b/packages/tracker-core/src/authentication/mod.rs index 12b742b8b..49e63666f 100644 --- a/packages/tracker-core/src/authentication/mod.rs +++ b/packages/tracker-core/src/authentication/mod.rs @@ -65,7 +65,7 @@ mod tests { config: &Configuration, ) -> (Arc, Arc) { let database = initialize_database(&config.core); - let db_key_repository = Arc::new(DatabaseKeyRepository::new(&database)); + let db_key_repository = Arc::new(DatabaseKeyRepository::new(database.auth_key_store())); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); let authentication_service = Arc::new(service::AuthenticationService::new(&config.core, &in_memory_key_repository)); let keys_handler = Arc::new(KeysHandler::new( diff --git a/packages/tracker-core/src/container.rs b/packages/tracker-core/src/container.rs index 93b8efd7e..ce00ecd08 100644 --- a/packages/tracker-core/src/container.rs +++ b/packages/tracker-core/src/container.rs @@ -9,7 +9,7 @@ use crate::authentication::key::repository::in_memory::InMemoryKeyRepository; use crate::authentication::key::repository::persisted::DatabaseKeyRepository; use crate::authentication::service::AuthenticationService; use crate::databases::setup::initialize_database; -use crate::databases::Database; +use crate::databases::Persistence; use crate::scrape_handler::ScrapeHandler; use crate::statistics::persisted::downloads::DatabaseDownloadsMetricRepository; use crate::torrent::manager::TorrentsManager; @@ -22,7 +22,7 @@ use crate::{statistics, whitelist}; pub struct TrackerCoreContainer { pub core_config: Arc, - pub database: Arc>, + pub database: Persistence, pub announce_handler: Arc, pub scrape_handler: Arc, pub keys_handler: Arc, @@ -45,8 +45,8 @@ impl TrackerCoreContainer { let database = initialize_database(core_config); let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_authorization = Arc::new(WhitelistAuthorization::new(core_config, &in_memory_whitelist.clone())); - let whitelist_manager = initialize_whitelist_manager(database.clone(), in_memory_whitelist.clone()); - let db_key_repository = Arc::new(DatabaseKeyRepository::new(&database)); + let whitelist_manager = initialize_whitelist_manager(database.whitelist_store(), in_memory_whitelist.clone()); + let db_key_repository = Arc::new(DatabaseKeyRepository::new(database.auth_key_store())); let in_memory_key_repository = Arc::new(InMemoryKeyRepository::default()); let authentication_service = Arc::new(AuthenticationService::new(core_config, &in_memory_key_repository)); let keys_handler = Arc::new(KeysHandler::new( @@ -56,7 +56,7 @@ impl TrackerCoreContainer { let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::new( swarm_coordination_registry_container.swarms.clone(), )); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let torrents_manager = Arc::new(TorrentsManager::new( core_config, diff --git a/packages/tracker-core/src/databases/driver/mod.rs b/packages/tracker-core/src/databases/driver/mod.rs index 6c849bb70..4f8d1f2ec 100644 --- a/packages/tracker-core/src/databases/driver/mod.rs +++ b/packages/tracker-core/src/databases/driver/mod.rs @@ -1,342 +1,374 @@ //! Database driver factory. +use std::sync::Arc; + use mysql::Mysql; +use postgres::Postgres; use serde::{Deserialize, Serialize}; use sqlite::Sqlite; use super::error::Error; -use super::Database; +use super::{AuthKeyStore, Persistence, SchemaMigrator, TorrentMetricsStore, WhitelistStore}; /// Metric name in DB for the total number of downloads across all torrents. -const TORRENTS_DOWNLOADS_TOTAL: &str = "torrents_downloads_total"; +pub const TORRENTS_DOWNLOADS_TOTAL: &str = "torrents_downloads_total"; /// The database management system used by the tracker. -/// -/// Refer to: -/// -/// - [Torrust Tracker Configuration](https://docs.rs/torrust-tracker-configuration). -/// - [Torrust Tracker](https://docs.rs/torrust-tracker). -/// -/// For more information about persistence. #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, derive_more::Display, Clone)] pub enum Driver { /// The Sqlite3 database driver. Sqlite3, /// The `MySQL` database driver. MySQL, + /// The `PostgreSQL` database driver. + PostgreSQL, } -/// It builds a new database driver. -/// -/// Example for `SQLite3`: -/// -/// ```text -/// use bittorrent_tracker_core::databases; -/// use bittorrent_tracker_core::databases::driver::Driver; -/// -/// let db_driver = Driver::Sqlite3; -/// let db_path = "./storage/tracker/lib/database/sqlite3.db".to_string(); -/// let database = databases::driver::build(&db_driver, &db_path); -/// ``` -/// -/// Example for `MySQL`: -/// -/// ```text -/// use bittorrent_tracker_core::databases; -/// use bittorrent_tracker_core::databases::driver::Driver; -/// -/// let db_driver = Driver::MySQL; -/// let db_path = "mysql://db_user:db_user_secret_password@mysql:3306/torrust_tracker".to_string(); -/// let database = databases::driver::build(&db_driver, &db_path); -/// ``` -/// -/// Refer to the [configuration documentation](https://docs.rs/torrust-tracker-configuration) -/// for more information about the database configuration. -/// -/// > **WARNING**: The driver instantiation runs database migrations. -/// -/// # Errors -/// -/// This function will return an error if unable to connect to the database. -/// -/// # Panics -/// -/// This function will panic if unable to create database tables. pub mod mysql; +pub mod postgres; pub mod sqlite; -/// It builds a new database driver. -/// -/// # Panics -/// -/// Will panic if unable to create database tables. +/// Builds a new persistence backend. /// /// # Errors /// -/// Will return `Error` if unable to build the driver. -pub(crate) fn build(driver: &Driver, db_path: &str) -> Result, Error> { - let database: Box = match driver { - Driver::Sqlite3 => Box::new(Sqlite::new(db_path)?), - Driver::MySQL => Box::new(Mysql::new(db_path)?), - }; - - database.create_database_tables().expect("Could not create database tables."); - - Ok(database) +/// Will return [`Error`] if unable to build the backend. +pub(crate) fn build(driver: &Driver, db_path: &str) -> Result { + match driver { + Driver::Sqlite3 => { + let backend = Arc::new(Sqlite::new(db_path)?); + Ok(Persistence::new( + backend.clone() as Arc, + backend.clone() as Arc, + backend.clone() as Arc, + backend as Arc, + )) + } + Driver::MySQL => { + let backend = Arc::new(Mysql::new(db_path)?); + Ok(Persistence::new( + backend.clone() as Arc, + backend.clone() as Arc, + backend.clone() as Arc, + backend as Arc, + )) + } + Driver::PostgreSQL => { + let backend = Arc::new(Postgres::new(db_path)?); + Ok(Persistence::new( + backend.clone() as Arc, + backend.clone() as Arc, + backend.clone() as Arc, + backend as Arc, + )) + } + } } #[cfg(test)] pub(crate) mod tests { - use std::sync::Arc; - use std::time::Duration; - - use crate::databases::Database; - - pub async fn run_tests(driver: &Arc>) { - // Since the interface is very simple and there are no conflicts between - // tests, we share the same database. If we want to isolate the tests in - // the future, we can create a new database for each test. + use super::Persistence; + pub async fn run_tests(driver: &Persistence) { database_setup(driver).await; - // Persistent torrents (stats) - - // Torrent metrics - handling_torrent_persistence::it_should_save_and_load_persistent_torrents(driver); - handling_torrent_persistence::it_should_load_all_persistent_torrents(driver); - handling_torrent_persistence::it_should_increase_the_number_of_downloads_for_a_given_torrent(driver); - // Aggregate metrics for all torrents - handling_torrent_persistence::it_should_save_and_load_the_global_number_of_downloads(driver); - handling_torrent_persistence::it_should_load_the_global_number_of_downloads(driver); - handling_torrent_persistence::it_should_increase_the_global_number_of_downloads(driver); - - // Authentication keys (for private trackers) - - handling_authentication_keys::it_should_load_the_keys(driver); - - // Permanent keys - handling_authentication_keys::it_should_save_and_load_permanent_authentication_keys(driver); - handling_authentication_keys::it_should_remove_a_permanent_authentication_key(driver); - - // Expiring keys - handling_authentication_keys::it_should_save_and_load_expiring_authentication_keys(driver); - handling_authentication_keys::it_should_remove_an_expiring_authentication_key(driver); - - // Whitelist (for listed trackers) - - handling_the_whitelist::it_should_load_the_whitelist(driver); - handling_the_whitelist::it_should_add_and_get_infohashes(driver); - handling_the_whitelist::it_should_remove_an_infohash_from_the_whitelist(driver); - handling_the_whitelist::it_should_fail_trying_to_add_the_same_infohash_twice(driver); + handling_torrent_persistence::it_should_save_and_load_persistent_torrents(driver).await; + handling_torrent_persistence::it_should_load_all_persistent_torrents(driver).await; + handling_torrent_persistence::it_should_increase_the_number_of_downloads_for_a_given_torrent(driver).await; + handling_torrent_persistence::it_should_save_and_load_the_global_number_of_downloads(driver).await; + handling_torrent_persistence::it_should_load_the_global_number_of_downloads(driver).await; + handling_torrent_persistence::it_should_increase_the_global_number_of_downloads(driver).await; + handling_torrent_persistence::it_should_support_large_download_counters(driver).await; + + handling_authentication_keys::it_should_load_the_keys(driver).await; + handling_authentication_keys::it_should_save_and_load_permanent_authentication_keys(driver).await; + handling_authentication_keys::it_should_save_and_load_expiring_authentication_keys(driver).await; + handling_authentication_keys::it_should_remove_a_permanent_authentication_key(driver).await; + handling_authentication_keys::it_should_remove_an_expiring_authentication_key(driver).await; + + handling_the_whitelist::it_should_load_the_whitelist(driver).await; + handling_the_whitelist::it_should_add_and_get_infohashes(driver).await; + handling_the_whitelist::it_should_remove_an_infohash_from_the_whitelist(driver).await; + handling_the_whitelist::it_should_fail_trying_to_add_the_same_infohash_twice(driver).await; } - /// It initializes the database schema. - /// - /// Since the drop SQL queries don't check if the tables already exist, - /// we have to create them first, and then drop them. - /// - /// The method to drop tables does not use "DROP TABLE IF EXISTS". We can - /// change this function when we update the `Database::drop_database_tables` - /// method to use "DROP TABLE IF EXISTS". - async fn database_setup(driver: &Arc>) { + async fn database_setup(driver: &Persistence) { create_database_tables(driver).await.expect("database tables creation failed"); - driver.drop_database_tables().expect("old database tables deletion failed"); + driver + .schema_migrator() + .drop_database_tables() + .await + .expect("old database tables deletion failed"); create_database_tables(driver) .await .expect("database tables creation from empty schema failed"); } - async fn create_database_tables(driver: &Arc>) -> Result<(), Box> { + async fn create_database_tables(driver: &Persistence) -> Result<(), Box> { + use std::time::Duration; + + let mut last_error = None; + for _ in 0..5 { - if driver.create_database_tables().is_ok() { - return Ok(()); + match driver.schema_migrator().create_database_tables().await { + Ok(()) => return Ok(()), + Err(err) => last_error = Some(err), } tokio::time::sleep(Duration::from_secs(2)).await; } - Err("Database is not ready after retries.".into()) + + match last_error { + Some(err) => Err(format!("Database is not ready after retries: {err}").into()), + None => Err("Database is not ready after retries.".into()), + } } mod handling_torrent_persistence { - - use std::sync::Arc; - - use crate::databases::Database; + use crate::databases::driver::tests::Persistence; use crate::test_helpers::tests::sample_info_hash; - // Metrics per torrent - - pub fn it_should_save_and_load_persistent_torrents(driver: &Arc>) { + pub async fn it_should_save_and_load_persistent_torrents(driver: &Persistence) { let infohash = sample_info_hash(); - let number_of_downloads = 1; - - driver.save_torrent_downloads(&infohash, number_of_downloads).unwrap(); + driver + .torrent_metrics_store() + .save_torrent_downloads(&infohash, 1) + .await + .unwrap(); - let number_of_downloads = driver.load_torrent_downloads(&infohash).unwrap().unwrap(); + let number_of_downloads = driver + .torrent_metrics_store() + .load_torrent_downloads(&infohash) + .await + .unwrap() + .unwrap(); assert_eq!(number_of_downloads, 1); } - pub fn it_should_load_all_persistent_torrents(driver: &Arc>) { + pub async fn it_should_load_all_persistent_torrents(driver: &Persistence) { let infohash = sample_info_hash(); - let number_of_downloads = 1; - - driver.save_torrent_downloads(&infohash, number_of_downloads).unwrap(); + driver + .torrent_metrics_store() + .save_torrent_downloads(&infohash, 1) + .await + .unwrap(); - let torrents = driver.load_all_torrents_downloads().unwrap(); + let torrents = driver.torrent_metrics_store().load_all_torrents_downloads().await.unwrap(); assert_eq!(torrents.len(), 1); - assert_eq!(torrents.get(&infohash), Some(number_of_downloads).as_ref()); + assert_eq!(torrents.get(&infohash), Some(1_u64).as_ref()); } - pub fn it_should_increase_the_number_of_downloads_for_a_given_torrent(driver: &Arc>) { + pub async fn it_should_increase_the_number_of_downloads_for_a_given_torrent(driver: &Persistence) { let infohash = sample_info_hash(); - let number_of_downloads = 1; - - driver.save_torrent_downloads(&infohash, number_of_downloads).unwrap(); - - driver.increase_downloads_for_torrent(&infohash).unwrap(); - - let number_of_downloads = driver.load_torrent_downloads(&infohash).unwrap().unwrap(); + driver + .torrent_metrics_store() + .save_torrent_downloads(&infohash, 1) + .await + .unwrap(); + + driver + .torrent_metrics_store() + .increase_downloads_for_torrent(&infohash) + .await + .unwrap(); + + let number_of_downloads = driver + .torrent_metrics_store() + .load_torrent_downloads(&infohash) + .await + .unwrap() + .unwrap(); assert_eq!(number_of_downloads, 2); } - // Aggregate metrics for all torrents - - pub fn it_should_save_and_load_the_global_number_of_downloads(driver: &Arc>) { - let number_of_downloads = 1; + pub async fn it_should_save_and_load_the_global_number_of_downloads(driver: &Persistence) { + driver.torrent_metrics_store().save_global_downloads(1).await.unwrap(); - driver.save_global_downloads(number_of_downloads).unwrap(); - - let number_of_downloads = driver.load_global_downloads().unwrap().unwrap(); + let number_of_downloads = driver.torrent_metrics_store().load_global_downloads().await.unwrap().unwrap(); assert_eq!(number_of_downloads, 1); } - pub fn it_should_load_the_global_number_of_downloads(driver: &Arc>) { - let number_of_downloads = 1; - - driver.save_global_downloads(number_of_downloads).unwrap(); + pub async fn it_should_load_the_global_number_of_downloads(driver: &Persistence) { + driver.torrent_metrics_store().save_global_downloads(1).await.unwrap(); - let number_of_downloads = driver.load_global_downloads().unwrap().unwrap(); + let number_of_downloads = driver.torrent_metrics_store().load_global_downloads().await.unwrap().unwrap(); assert_eq!(number_of_downloads, 1); } - pub fn it_should_increase_the_global_number_of_downloads(driver: &Arc>) { - let number_of_downloads = 1; + pub async fn it_should_increase_the_global_number_of_downloads(driver: &Persistence) { + driver.torrent_metrics_store().save_global_downloads(1).await.unwrap(); - driver.save_global_downloads(number_of_downloads).unwrap(); + driver.torrent_metrics_store().increase_global_downloads().await.unwrap(); - driver.increase_global_downloads().unwrap(); - - let number_of_downloads = driver.load_global_downloads().unwrap().unwrap(); + let number_of_downloads = driver.torrent_metrics_store().load_global_downloads().await.unwrap().unwrap(); assert_eq!(number_of_downloads, 2); } + + pub async fn it_should_support_large_download_counters(driver: &Persistence) { + let infohash = sample_info_hash(); + let large_value = u64::from(u32::MAX); + + driver + .torrent_metrics_store() + .save_torrent_downloads(&infohash, large_value) + .await + .unwrap(); + driver + .torrent_metrics_store() + .save_global_downloads(large_value) + .await + .unwrap(); + + assert_eq!( + driver + .torrent_metrics_store() + .load_torrent_downloads(&infohash) + .await + .unwrap(), + Some(large_value) + ); + assert_eq!( + driver.torrent_metrics_store().load_global_downloads().await.unwrap(), + Some(large_value) + ); + } } mod handling_authentication_keys { - - use std::sync::Arc; use std::time::Duration; use crate::authentication::key::{generate_expiring_key, generate_permanent_key}; - use crate::databases::Database; + use crate::databases::driver::tests::Persistence; - pub fn it_should_load_the_keys(driver: &Arc>) { + pub async fn it_should_load_the_keys(driver: &Persistence) { let permanent_peer_key = generate_permanent_key(); - driver.add_key_to_keys(&permanent_peer_key).unwrap(); + driver.auth_key_store().add_key_to_keys(&permanent_peer_key).await.unwrap(); let expiring_peer_key = generate_expiring_key(Duration::from_secs(120)); - driver.add_key_to_keys(&expiring_peer_key).unwrap(); + driver.auth_key_store().add_key_to_keys(&expiring_peer_key).await.unwrap(); - let keys = driver.load_keys().unwrap(); + let keys = driver.auth_key_store().load_keys().await.unwrap(); assert!(keys.contains(&permanent_peer_key)); assert!(keys.contains(&expiring_peer_key)); } - pub fn it_should_save_and_load_permanent_authentication_keys(driver: &Arc>) { + pub async fn it_should_save_and_load_permanent_authentication_keys(driver: &Persistence) { let peer_key = generate_permanent_key(); - driver.add_key_to_keys(&peer_key).unwrap(); + driver.auth_key_store().add_key_to_keys(&peer_key).await.unwrap(); - let stored_peer_key = driver.get_key_from_keys(&peer_key.key()).unwrap().unwrap(); + let stored_peer_key = driver + .auth_key_store() + .get_key_from_keys(&peer_key.key()) + .await + .unwrap() + .unwrap(); assert_eq!(stored_peer_key, peer_key); } - pub fn it_should_save_and_load_expiring_authentication_keys(driver: &Arc>) { + pub async fn it_should_save_and_load_expiring_authentication_keys(driver: &Persistence) { let peer_key = generate_expiring_key(Duration::from_secs(120)); - driver.add_key_to_keys(&peer_key).unwrap(); + driver.auth_key_store().add_key_to_keys(&peer_key).await.unwrap(); - let stored_peer_key = driver.get_key_from_keys(&peer_key.key()).unwrap().unwrap(); + let stored_peer_key = driver + .auth_key_store() + .get_key_from_keys(&peer_key.key()) + .await + .unwrap() + .unwrap(); assert_eq!(stored_peer_key, peer_key); assert_eq!(stored_peer_key.expiry_time(), peer_key.expiry_time()); } - pub fn it_should_remove_a_permanent_authentication_key(driver: &Arc>) { + pub async fn it_should_remove_a_permanent_authentication_key(driver: &Persistence) { let peer_key = generate_permanent_key(); - driver.add_key_to_keys(&peer_key).unwrap(); + driver.auth_key_store().add_key_to_keys(&peer_key).await.unwrap(); - driver.remove_key_from_keys(&peer_key.key()).unwrap(); + driver.auth_key_store().remove_key_from_keys(&peer_key.key()).await.unwrap(); - assert!(driver.get_key_from_keys(&peer_key.key()).unwrap().is_none()); + assert!(driver + .auth_key_store() + .get_key_from_keys(&peer_key.key()) + .await + .unwrap() + .is_none()); } - pub fn it_should_remove_an_expiring_authentication_key(driver: &Arc>) { + pub async fn it_should_remove_an_expiring_authentication_key(driver: &Persistence) { let peer_key = generate_expiring_key(Duration::from_secs(120)); - driver.add_key_to_keys(&peer_key).unwrap(); + driver.auth_key_store().add_key_to_keys(&peer_key).await.unwrap(); - driver.remove_key_from_keys(&peer_key.key()).unwrap(); + driver.auth_key_store().remove_key_from_keys(&peer_key.key()).await.unwrap(); - assert!(driver.get_key_from_keys(&peer_key.key()).unwrap().is_none()); + assert!(driver + .auth_key_store() + .get_key_from_keys(&peer_key.key()) + .await + .unwrap() + .is_none()); } } mod handling_the_whitelist { - - use std::sync::Arc; - - use crate::databases::Database; + use crate::databases::driver::tests::Persistence; use crate::test_helpers::tests::random_info_hash; - pub fn it_should_load_the_whitelist(driver: &Arc>) { + pub async fn it_should_load_the_whitelist(driver: &Persistence) { let infohash = random_info_hash(); - driver.add_info_hash_to_whitelist(infohash).unwrap(); + driver.whitelist_store().add_info_hash_to_whitelist(infohash).await.unwrap(); - let whitelist = driver.load_whitelist().unwrap(); + let whitelist = driver.whitelist_store().load_whitelist().await.unwrap(); assert!(whitelist.contains(&infohash)); } - pub fn it_should_add_and_get_infohashes(driver: &Arc>) { + pub async fn it_should_add_and_get_infohashes(driver: &Persistence) { let infohash = random_info_hash(); - driver.add_info_hash_to_whitelist(infohash).unwrap(); + driver.whitelist_store().add_info_hash_to_whitelist(infohash).await.unwrap(); - let stored_infohash = driver.get_info_hash_from_whitelist(infohash).unwrap().unwrap(); + let stored_infohash = driver + .whitelist_store() + .get_info_hash_from_whitelist(infohash) + .await + .unwrap() + .unwrap(); assert_eq!(stored_infohash, infohash); } - pub fn it_should_remove_an_infohash_from_the_whitelist(driver: &Arc>) { + pub async fn it_should_remove_an_infohash_from_the_whitelist(driver: &Persistence) { let infohash = random_info_hash(); - driver.add_info_hash_to_whitelist(infohash).unwrap(); - - driver.remove_info_hash_from_whitelist(infohash).unwrap(); - - assert!(driver.get_info_hash_from_whitelist(infohash).unwrap().is_none()); + driver.whitelist_store().add_info_hash_to_whitelist(infohash).await.unwrap(); + + driver + .whitelist_store() + .remove_info_hash_from_whitelist(infohash) + .await + .unwrap(); + + assert!(driver + .whitelist_store() + .get_info_hash_from_whitelist(infohash) + .await + .unwrap() + .is_none()); } - pub fn it_should_fail_trying_to_add_the_same_infohash_twice(driver: &Arc>) { + pub async fn it_should_fail_trying_to_add_the_same_infohash_twice(driver: &Persistence) { let infohash = random_info_hash(); - driver.add_info_hash_to_whitelist(infohash).unwrap(); - let result = driver.add_info_hash_to_whitelist(infohash); + driver.whitelist_store().add_info_hash_to_whitelist(infohash).await.unwrap(); + let result = driver.whitelist_store().add_info_hash_to_whitelist(infohash).await; assert!(result.is_err()); } diff --git a/packages/tracker-core/src/databases/driver/mysql.rs b/packages/tracker-core/src/databases/driver/mysql.rs index da2f86ce8..af887b646 100644 --- a/packages/tracker-core/src/databases/driver/mysql.rs +++ b/packages/tracker-core/src/databases/driver/mysql.rs @@ -1,388 +1,458 @@ //! The `MySQL` database driver. -//! -//! This module provides an implementation of the [`Database`] trait for `MySQL` -//! using the `r2d2_mysql` connection pool. It configures the MySQL connection -//! based on a URL, creates the necessary tables (for torrent metrics, torrent -//! whitelist, and authentication keys), and implements all CRUD operations -//! required by the persistence layer. use std::str::FromStr; -use std::time::Duration; +use std::sync::atomic::{AtomicBool, Ordering}; +use async_trait::async_trait; use bittorrent_primitives::info_hash::InfoHash; -use r2d2::Pool; -use r2d2_mysql::mysql::prelude::Queryable; -use r2d2_mysql::mysql::{params, Opts, OptsBuilder}; -use r2d2_mysql::MySqlConnectionManager; -use torrust_tracker_primitives::{NumberOfDownloads, NumberOfDownloadsBTreeMap}; - -use super::{Database, Driver, Error, TORRENTS_DOWNLOADS_TOTAL}; -use crate::authentication::key::AUTH_KEY_LENGTH; +use sqlx::migrate::Migrator; +use sqlx::mysql::{MySqlConnectOptions, MySqlPoolOptions}; +use sqlx::{ConnectOptions, MySqlPool, Row}; +use tokio::sync::Mutex; +use torrust_tracker_primitives::{DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; + +use super::{Driver, TORRENTS_DOWNLOADS_TOTAL}; use crate::authentication::{self, Key}; +use crate::databases::error::Error; +use crate::databases::{AuthKeyStore, SchemaMigrator, TorrentMetricsStore, WhitelistStore}; const DRIVER: Driver = Driver::MySQL; +static MIGRATOR: Migrator = sqlx::migrate!("migrations/mysql"); -/// `MySQL` driver implementation. -/// -/// This struct encapsulates a connection pool for `MySQL`, built using the -/// `r2d2_mysql` connection manager. It implements the [`Database`] trait to -/// provide persistence operations. +/// `MySQL` driver implementation backed by `sqlx`. pub(crate) struct Mysql { - pool: Pool, + pool: MySqlPool, + schema_ready: AtomicBool, + schema_lock: Mutex<()>, } impl Mysql { - /// It instantiates a new `MySQL` database driver. - /// - /// Refer to [`databases::Database::new`](crate::core::databases::Database::new). + /// Instantiates a new `MySQL` database driver. /// /// # Errors /// - /// Will return `r2d2::Error` if `db_path` is not able to create `MySQL` database. + /// Returns an [`Error`] if the database URL cannot be parsed. pub fn new(db_path: &str) -> Result { - let opts = Opts::from_url(db_path)?; - let builder = OptsBuilder::from_opts(opts); - let manager = MySqlConnectionManager::new(builder); - let pool = r2d2::Pool::builder().build(manager).map_err(|e| (e, DRIVER))?; + let options = MySqlConnectOptions::from_str(db_path) + .map_err(|err| Error::connection_error(DRIVER, err))? + .disable_statement_logging(); + + let pool = MySqlPoolOptions::new().connect_lazy_with(options); - Ok(Self { pool }) + Ok(Self { + pool, + schema_ready: AtomicBool::new(false), + schema_lock: Mutex::new(()), + }) } - fn load_torrent_aggregate_metric(&self, metric_name: &str) -> Result, Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn ensure_schema(&self) -> Result<(), Error> { + if self.schema_ready.load(Ordering::Acquire) { + return Ok(()); + } - let query = conn.exec_first::( - "SELECT value FROM torrent_aggregate_metrics WHERE metric_name = :metric_name", - params! { "metric_name" => metric_name }, - ); + let _guard = self.schema_lock.lock().await; - let persistent_torrent = query?; + if self.schema_ready.load(Ordering::Acquire) { + return Ok(()); + } - Ok(persistent_torrent) + self.run_migrations().await } - fn save_torrent_aggregate_metric(&self, metric_name: &str, completed: NumberOfDownloads) -> Result<(), Error> { - const COMMAND : &str = "INSERT INTO torrent_aggregate_metrics (metric_name, value) VALUES (:metric_name, :completed) ON DUPLICATE KEY UPDATE value = VALUES(value)"; + async fn run_migrations(&self) -> Result<(), Error> { + MIGRATOR + .run(&self.pool) + .await + .map_err(|err| Error::migration_error(DRIVER, err))?; - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + self.schema_ready.store(true, Ordering::Release); - Ok(conn.exec_drop(COMMAND, params! { metric_name, completed })?) + Ok(()) } -} - -impl Database for Mysql { - /// Refer to [`databases::Database::create_database_tables`](crate::core::databases::Database::create_database_tables). - fn create_database_tables(&self) -> Result<(), Error> { - let create_whitelist_table = " - CREATE TABLE IF NOT EXISTS whitelist ( - id integer PRIMARY KEY AUTO_INCREMENT, - info_hash VARCHAR(40) NOT NULL UNIQUE - );" - .to_string(); - - let create_torrents_table = " - CREATE TABLE IF NOT EXISTS torrents ( - id integer PRIMARY KEY AUTO_INCREMENT, - info_hash VARCHAR(40) NOT NULL UNIQUE, - completed INTEGER DEFAULT 0 NOT NULL - );" - .to_string(); - - let create_torrent_aggregate_metrics_table = " - CREATE TABLE IF NOT EXISTS torrent_aggregate_metrics ( - id integer PRIMARY KEY AUTO_INCREMENT, - metric_name VARCHAR(50) NOT NULL UNIQUE, - value INTEGER DEFAULT 0 NOT NULL - );" - .to_string(); - - let create_keys_table = format!( - " - CREATE TABLE IF NOT EXISTS `keys` ( - `id` INT NOT NULL AUTO_INCREMENT, - `key` VARCHAR({}) NOT NULL, - `valid_until` INT(10), - PRIMARY KEY (`id`), - UNIQUE (`key`) - );", - i8::try_from(AUTH_KEY_LENGTH).expect("authentication key length should fit within a i8!") - ); - - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - conn.query_drop(&create_torrents_table) - .expect("Could not create torrents table."); - conn.query_drop(&create_torrent_aggregate_metrics_table) - .expect("Could not create create_torrent_aggregate_metrics_table table."); - conn.query_drop(&create_keys_table).expect("Could not create keys table."); - conn.query_drop(&create_whitelist_table) - .expect("Could not create whitelist table."); + fn decode_counter_i64(&self, value: i64) -> Result { + u64::try_from(value).map_err(|err| Error::invalid_query(DRIVER, err)) + } - Ok(()) + fn encode_counter(&self, value: NumberOfDownloads) -> Result { + i64::try_from(value).map_err(|err| Error::invalid_query(DRIVER, err)) } - /// Refer to [`databases::Database::drop_database_tables`](crate::core::databases::Database::drop_database_tables). - fn drop_database_tables(&self) -> Result<(), Error> { - let drop_whitelist_table = " - DROP TABLE `whitelist`;" - .to_string(); + fn decode_info_hash(&self, value: String) -> Result { + InfoHash::from_str(&value).map_err(|err| { + Error::invalid_query( + DRIVER, + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{err:?}")), + ) + }) + } - let drop_torrents_table = " - DROP TABLE `torrents`;" - .to_string(); + fn decode_key(&self, value: String) -> Result { + value.parse::().map_err(|err| Error::invalid_query(DRIVER, err)) + } - let drop_keys_table = " - DROP TABLE `keys`;" - .to_string(); + fn decode_valid_until(&self, value: Option) -> Result, Error> { + value + .map(|seconds| { + u64::try_from(seconds) + .map(DurationSinceUnixEpoch::from_secs) + .map_err(|err| Error::invalid_query(DRIVER, err)) + }) + .transpose() + } +} - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; +#[async_trait] +impl SchemaMigrator for Mysql { + async fn create_database_tables(&self) -> Result<(), Error> { + self.run_migrations().await + } - conn.query_drop(&drop_whitelist_table) - .expect("Could not drop `whitelist` table."); - conn.query_drop(&drop_torrents_table) - .expect("Could not drop `torrents` table."); - conn.query_drop(&drop_keys_table).expect("Could not drop `keys` table."); + async fn drop_database_tables(&self) -> Result<(), Error> { + // Tests intentionally use an explicit drop/create cycle to simulate failures. + // Callers that need a working schema after this must invoke `create_database_tables`. + sqlx::query("DROP TABLE IF EXISTS whitelist") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS torrent_aggregate_metrics") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS torrents") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS `keys`") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS _sqlx_migrations") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; Ok(()) } +} - /// Refer to [`databases::Database::load_persistent_torrents`](crate::core::databases::Database::load_persistent_torrents). - fn load_all_torrents_downloads(&self) -> Result { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; +#[async_trait] +impl TorrentMetricsStore for Mysql { + async fn load_all_torrents_downloads(&self) -> Result { + self.ensure_schema().await?; - let torrents = conn.query_map( - "SELECT info_hash, completed FROM torrents", - |(info_hash_string, completed): (String, u32)| { - let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); - (info_hash, completed) - }, - )?; + let rows = sqlx::query("SELECT info_hash, completed FROM torrents") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(torrents.iter().copied().collect()) - } + let mut torrents = NumberOfDownloadsBTreeMap::new(); - /// Refer to [`databases::Database::load_persistent_torrent`](crate::core::databases::Database::load_persistent_torrent). - fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + for row in rows { + let info_hash_string: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + let completed: i64 = row.try_get("completed").map_err(|err| (err, DRIVER))?; - let query = conn.exec_first::( - "SELECT completed FROM torrents WHERE info_hash = :info_hash", - params! { "info_hash" => info_hash.to_hex_string() }, - ); - - let persistent_torrent = query?; + torrents.insert(self.decode_info_hash(info_hash_string)?, self.decode_counter_i64(completed)?); + } - Ok(persistent_torrent) + Ok(torrents) } - /// Refer to [`databases::Database::save_persistent_torrent`](crate::core::databases::Database::save_persistent_torrent). - fn save_torrent_downloads(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error> { - const COMMAND : &str = "INSERT INTO torrents (info_hash, completed) VALUES (:info_hash_str, :completed) ON DUPLICATE KEY UPDATE completed = VALUES(completed)"; - - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { + self.ensure_schema().await?; - let info_hash_str = info_hash.to_string(); + let row = sqlx::query("SELECT completed FROM torrents WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(conn.exec_drop(COMMAND, params! { info_hash_str, completed })?) + row.map(|row| { + let completed: i64 = row.try_get("completed").map_err(|err| (err, DRIVER))?; + self.decode_counter_i64(completed) + }) + .transpose() } - /// Refer to [`databases::Database::increase_number_of_downloads`](crate::core::databases::Database::increase_number_of_downloads). - fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.ensure_schema().await?; + + let encoded_downloaded = self.encode_counter(downloaded)?; + + let insert = sqlx::query( + "INSERT INTO torrents (info_hash, completed) VALUES (?, ?) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", + ) + .bind(info_hash.to_hex_string()) + .bind(encoded_downloaded) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(()) + } + } - let info_hash_str = info_hash.to_string(); + async fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { + self.ensure_schema().await?; - conn.exec_drop( - "UPDATE torrents SET completed = completed + 1 WHERE info_hash = :info_hash_str", - params! { info_hash_str }, - )?; + sqlx::query("UPDATE torrents SET completed = completed + 1 WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; Ok(()) } - /// Refer to [`databases::Database::load_global_number_of_downloads`](crate::core::databases::Database::load_global_number_of_downloads). - fn load_global_downloads(&self) -> Result, Error> { - self.load_torrent_aggregate_metric(TORRENTS_DOWNLOADS_TOTAL) - } + async fn load_global_downloads(&self) -> Result, Error> { + self.ensure_schema().await?; + + let row = sqlx::query("SELECT value FROM torrent_aggregate_metrics WHERE metric_name = ?") + .bind(TORRENTS_DOWNLOADS_TOTAL) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - /// Refer to [`databases::Database::save_global_number_of_downloads`](crate::core::databases::Database::save_global_number_of_downloads). - fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error> { - self.save_torrent_aggregate_metric(TORRENTS_DOWNLOADS_TOTAL, downloaded) + row.map(|row| { + let value: i64 = row.try_get("value").map_err(|err| (err, DRIVER))?; + self.decode_counter_i64(value) + }) + .transpose() } - /// Refer to [`databases::Database::increase_global_number_of_downloads`](crate::core::databases::Database::increase_global_number_of_downloads). - fn increase_global_downloads(&self) -> Result<(), Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.ensure_schema().await?; + + let encoded_downloaded = self.encode_counter(downloaded)?; + + let insert = sqlx::query( + "INSERT INTO torrent_aggregate_metrics (metric_name, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value = VALUES(value)", + ) + .bind(TORRENTS_DOWNLOADS_TOTAL) + .bind(encoded_downloaded) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(()) + } + } - let metric_name = TORRENTS_DOWNLOADS_TOTAL; + async fn increase_global_downloads(&self) -> Result<(), Error> { + self.ensure_schema().await?; - conn.exec_drop( - "UPDATE torrent_aggregate_metrics SET value = value + 1 WHERE metric_name = :metric_name", - params! { metric_name }, - )?; + sqlx::query("UPDATE torrent_aggregate_metrics SET value = value + 1 WHERE metric_name = ?") + .bind(TORRENTS_DOWNLOADS_TOTAL) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; Ok(()) } +} - /// Refer to [`databases::Database::load_keys`](crate::core::databases::Database::load_keys). - fn load_keys(&self) -> Result, Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let keys = conn.query_map( - "SELECT `key`, valid_until FROM `keys`", - |(key, valid_until): (String, Option)| match valid_until { - Some(valid_until) => authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: Some(Duration::from_secs(valid_until.unsigned_abs())), - }, - None => authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: None, - }, - }, - )?; - - Ok(keys) +#[async_trait] +impl WhitelistStore for Mysql { + async fn load_whitelist(&self) -> Result, Error> { + self.ensure_schema().await?; + + let rows = sqlx::query("SELECT info_hash FROM whitelist") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + rows.into_iter() + .map(|row| { + let info_hash: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + self.decode_info_hash(info_hash) + }) + .collect() } - /// Refer to [`databases::Database::load_whitelist`](crate::core::databases::Database::load_whitelist). - fn load_whitelist(&self) -> Result, Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { + self.ensure_schema().await?; - let info_hashes = conn.query_map("SELECT info_hash FROM whitelist", |info_hash: String| { - InfoHash::from_str(&info_hash).unwrap() - })?; + let row = sqlx::query("SELECT info_hash FROM whitelist WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(info_hashes) + row.map(|row| { + let value: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + self.decode_info_hash(value) + }) + .transpose() } - /// Refer to [`databases::Database::get_info_hash_from_whitelist`](crate::core::databases::Database::get_info_hash_from_whitelist). - fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let select = conn.exec_first::( - "SELECT info_hash FROM whitelist WHERE info_hash = :info_hash", - params! { "info_hash" => info_hash.to_hex_string() }, - )?; - - let info_hash = select.map(|f| InfoHash::from_str(&f).expect("Failed to decode InfoHash String from DB!")); - - Ok(info_hash) + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { + self.ensure_schema().await?; + + let insert = sqlx::query("INSERT INTO whitelist (info_hash) VALUES (?)") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(insert.rows_affected() as usize) + } } - /// Refer to [`databases::Database::add_info_hash_to_whitelist`](crate::core::databases::Database::add_info_hash_to_whitelist). - fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let info_hash_str = info_hash.to_string(); - - conn.exec_drop( - "INSERT INTO whitelist (info_hash) VALUES (:info_hash_str)", - params! { info_hash_str }, - )?; - - Ok(1) + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { + self.ensure_schema().await?; + + let deleted = sqlx::query("DELETE FROM whitelist WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + let deleted = deleted.rows_affected() as usize; + + if deleted == 1 { + Ok(deleted) + } else { + Err(Error::DeleteFailed { + location: std::panic::Location::caller(), + error_code: deleted, + driver: DRIVER, + }) + } } +} - /// Refer to [`databases::Database::remove_info_hash_from_whitelist`](crate::core::databases::Database::remove_info_hash_from_whitelist). - fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let info_hash = info_hash.to_string(); - - conn.exec_drop("DELETE FROM whitelist WHERE info_hash = :info_hash", params! { info_hash })?; - - Ok(1) +#[async_trait] +impl AuthKeyStore for Mysql { + async fn load_keys(&self) -> Result, Error> { + self.ensure_schema().await?; + + let rows = sqlx::query("SELECT `key`, valid_until FROM `keys`") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + rows.into_iter() + .map(|row| { + let key: String = row.try_get("key").map_err(|err| (err, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|err| (err, DRIVER))?; + + Ok(authentication::PeerKey { + key: self.decode_key(key)?, + valid_until: self.decode_valid_until(valid_until)?, + }) + }) + .collect() } - /// Refer to [`databases::Database::get_key_from_keys`](crate::core::databases::Database::get_key_from_keys). - fn get_key_from_keys(&self, key: &Key) -> Result, Error> { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let query = conn.exec_first::<(String, Option), _, _>( - "SELECT `key`, valid_until FROM `keys` WHERE `key` = :key", - params! { "key" => key.to_string() }, - ); + async fn get_key_from_keys(&self, key: &Key) -> Result, Error> { + self.ensure_schema().await?; + + let row = sqlx::query("SELECT `key`, valid_until FROM `keys` WHERE `key` = ?") + .bind(key.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + row.map(|row| { + let key: String = row.try_get("key").map_err(|err| (err, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|err| (err, DRIVER))?; + + Ok(authentication::PeerKey { + key: self.decode_key(key)?, + valid_until: self.decode_valid_until(valid_until)?, + }) + }) + .transpose() + } - let key = query?; - - Ok(key.map(|(key, opt_valid_until)| match opt_valid_until { - Some(valid_until) => authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: Some(Duration::from_secs(valid_until.unsigned_abs())), - }, - None => authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: None, - }, - })) - } - - /// Refer to [`databases::Database::add_key_to_keys`](crate::core::databases::Database::add_key_to_keys). - fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - match auth_key.valid_until { - Some(valid_until) => conn.exec_drop( - "INSERT INTO `keys` (`key`, valid_until) VALUES (:key, :valid_until)", - params! { "key" => auth_key.key.to_string(), "valid_until" => valid_until.as_secs().to_string() }, - )?, - None => conn.exec_drop( - "INSERT INTO `keys` (`key`) VALUES (:key)", - params! { "key" => auth_key.key.to_string() }, - )?, + async fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result { + self.ensure_schema().await?; + + let valid_until = auth_key + .valid_until + .map(|valid_until| valid_until.as_secs()) + .map(i64::try_from) + .transpose() + .map_err(|err| Error::invalid_query(DRIVER, err))?; + + let insert = sqlx::query("INSERT INTO `keys` (`key`, valid_until) VALUES (?, ?)") + .bind(auth_key.key.to_string()) + .bind(valid_until) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(insert.rows_affected() as usize) } - - Ok(1) } - /// Refer to [`databases::Database::remove_key_from_keys`](crate::core::databases::Database::remove_key_from_keys). - fn remove_key_from_keys(&self, key: &Key) -> Result { - let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - conn.exec_drop("DELETE FROM `keys` WHERE `key` = :key", params! { "key" => key.to_string() })?; - - Ok(1) + async fn remove_key_from_keys(&self, key: &Key) -> Result { + self.ensure_schema().await?; + + let deleted = sqlx::query("DELETE FROM `keys` WHERE `key` = ?") + .bind(key.to_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + let deleted = deleted.rows_affected() as usize; + + if deleted == 1 { + Ok(deleted) + } else { + Err(Error::DeleteFailed { + location: std::panic::Location::caller(), + error_code: deleted, + driver: DRIVER, + }) + } } } #[cfg(test)] mod tests { - use std::sync::Arc; - use testcontainers::core::IntoContainerPort; - /* - We run a MySQL container and run all the tests against the same container and database. - - Test for this driver are executed with: - - `TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST=true cargo test` - - The `Database` trait is very simple and we only have one driver that needs - a container. In the future we might want to use different approaches like: - - - https://github.com/testcontainers/testcontainers-rs/issues/707 - - https://www.infinyon.com/blog/2021/04/rust-custom-test-harness/ - - https://github.com/torrust/torrust-tracker/blob/develop/src/bin/e2e_tests_runner.rs - - If we increase the number of methods or the number or drivers. - */ use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use torrust_tracker_configuration::Core; - use super::Mysql; use crate::databases::driver::tests::run_tests; - use crate::databases::Database; + use crate::databases::driver::{build, Driver}; #[derive(Debug, Default)] struct StoppedMysqlContainer {} impl StoppedMysqlContainer { async fn run(self, config: &MysqlConfiguration) -> Result> { - let container = GenericImage::new("mysql", "8.0") + let image_name = std::env::var("TORRUST_TRACKER_CORE_MYSQL_DRIVER_IMAGE").unwrap_or_else(|_| "mysql".to_string()); + let image_tag = std::env::var("TORRUST_TRACKER_CORE_MYSQL_DRIVER_IMAGE_TAG").unwrap_or_else(|_| "8.0".to_string()); + + let container = GenericImage::new(image_name, image_tag) .with_exposed_port(config.internal_port.tcp()) - // todo: this does not work - //.with_wait_for(WaitFor::message_on_stdout("ready for connections")) .with_env_var("MYSQL_ROOT_PASSWORD", config.db_root_password.clone()) .with_env_var("MYSQL_DATABASE", config.database.clone()) .with_env_var("MYSQL_ROOT_HOST", "%") @@ -440,20 +510,15 @@ mod tests { fn core_configuration(host: &url::Host, port: u16, mysql_configuration: &MysqlConfiguration) -> Core { let mut config = Core::default(); - let database = mysql_configuration.database.clone(); - let db_user = mysql_configuration.db_user.clone(); - let db_password = mysql_configuration.db_root_password.clone(); - - config.database.path = format!("mysql://{db_user}:{db_password}@{host}:{port}/{database}"); + config.database.driver = torrust_tracker_configuration::Driver::MySQL; + config.database.path = format!( + "mysql://{}:{}@{}:{}/{}", + mysql_configuration.db_user, mysql_configuration.db_root_password, host, port, mysql_configuration.database + ); config } - fn initialize_driver(config: &Core) -> Arc> { - let driver: Arc> = Arc::new(Box::new(Mysql::new(&config.database.path).unwrap())); - driver - } - #[tokio::test] async fn run_mysql_driver_tests() -> Result<(), Box> { if std::env::var("TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST").is_err() { @@ -462,17 +527,14 @@ mod tests { } let mysql_configuration = MysqlConfiguration::default(); - let stopped_mysql_container = StoppedMysqlContainer::default(); - let mysql_container = stopped_mysql_container.run(&mysql_configuration).await.unwrap(); let host = mysql_container.get_host().await; let port = mysql_container.get_host_port_ipv4().await; let config = core_configuration(&host, port, &mysql_configuration); - - let driver = initialize_driver(&config); + let driver = build(&Driver::MySQL, &config.database.path)?; run_tests(&driver).await; diff --git a/packages/tracker-core/src/databases/driver/postgres.rs b/packages/tracker-core/src/databases/driver/postgres.rs new file mode 100644 index 000000000..78175ac2a --- /dev/null +++ b/packages/tracker-core/src/databases/driver/postgres.rs @@ -0,0 +1,563 @@ +//! The `PostgreSQL` database driver. +use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; +use bittorrent_primitives::info_hash::InfoHash; +use sqlx::migrate::Migrator; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, PgPool, Row}; +use tokio::sync::Mutex; +use torrust_tracker_primitives::{DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; + +use super::{Driver, TORRENTS_DOWNLOADS_TOTAL}; +use crate::authentication::{self, Key}; +use crate::databases::error::Error; +use crate::databases::{AuthKeyStore, SchemaMigrator, TorrentMetricsStore, WhitelistStore}; + +const DRIVER: Driver = Driver::PostgreSQL; +static MIGRATOR: Migrator = sqlx::migrate!("migrations/postgresql"); + +/// `PostgreSQL` driver implementation backed by `sqlx`. +pub(crate) struct Postgres { + pool: PgPool, + schema_ready: AtomicBool, + schema_lock: Mutex<()>, +} + +impl Postgres { + /// Instantiates a new `PostgreSQL` database driver. + /// + /// # Errors + /// + /// Returns an [`Error`] if the database URL cannot be parsed. + pub fn new(db_path: &str) -> Result { + let options = PgConnectOptions::from_str(db_path) + .map_err(|err| Error::connection_error(DRIVER, err))? + .disable_statement_logging(); + + let pool = PgPoolOptions::new().connect_lazy_with(options); + + Ok(Self { + pool, + schema_ready: AtomicBool::new(false), + schema_lock: Mutex::new(()), + }) + } + + async fn ensure_schema(&self) -> Result<(), Error> { + if self.schema_ready.load(Ordering::Acquire) { + return Ok(()); + } + + let _guard = self.schema_lock.lock().await; + + if self.schema_ready.load(Ordering::Acquire) { + return Ok(()); + } + + self.run_migrations().await + } + + async fn run_migrations(&self) -> Result<(), Error> { + MIGRATOR + .run(&self.pool) + .await + .map_err(|err| Error::migration_error(DRIVER, err))?; + + self.schema_ready.store(true, Ordering::Release); + + Ok(()) + } + + fn decode_counter_i64(&self, value: i64) -> Result { + u64::try_from(value).map_err(|err| Error::invalid_query(DRIVER, err)) + } + + fn encode_counter(&self, value: NumberOfDownloads) -> Result { + i64::try_from(value).map_err(|err| Error::invalid_query(DRIVER, err)) + } + + fn decode_info_hash(&self, value: String) -> Result { + InfoHash::from_str(&value).map_err(|err| { + Error::invalid_query( + DRIVER, + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{err:?}")), + ) + }) + } + + fn decode_key(&self, value: String) -> Result { + value.parse::().map_err(|err| Error::invalid_query(DRIVER, err)) + } + + fn decode_valid_until(&self, value: Option) -> Result, Error> { + value + .map(|seconds| { + u64::try_from(seconds) + .map(DurationSinceUnixEpoch::from_secs) + .map_err(|err| Error::invalid_query(DRIVER, err)) + }) + .transpose() + } +} + +#[async_trait] +impl SchemaMigrator for Postgres { + async fn create_database_tables(&self) -> Result<(), Error> { + self.run_migrations().await + } + + async fn drop_database_tables(&self) -> Result<(), Error> { + // Tests intentionally use an explicit drop/create cycle to simulate failures. + // Callers that need a working schema after this must invoke `create_database_tables`. + sqlx::query("DROP TABLE IF EXISTS whitelist") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS torrent_aggregate_metrics") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS torrents") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS keys") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS _sqlx_migrations") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + Ok(()) + } +} + +#[async_trait] +impl TorrentMetricsStore for Postgres { + async fn load_all_torrents_downloads(&self) -> Result { + self.ensure_schema().await?; + + let rows = sqlx::query("SELECT info_hash, completed FROM torrents") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + let mut torrents = NumberOfDownloadsBTreeMap::new(); + + for row in rows { + let info_hash_string: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + let completed: i64 = row.try_get("completed").map_err(|err| (err, DRIVER))?; + + torrents.insert(self.decode_info_hash(info_hash_string)?, self.decode_counter_i64(completed)?); + } + + Ok(torrents) + } + + async fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { + self.ensure_schema().await?; + + let row = sqlx::query("SELECT completed FROM torrents WHERE info_hash = $1") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + row.map(|row| { + let completed: i64 = row.try_get("completed").map_err(|err| (err, DRIVER))?; + self.decode_counter_i64(completed) + }) + .transpose() + } + + async fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.ensure_schema().await?; + + let encoded_downloaded = self.encode_counter(downloaded)?; + + let insert = sqlx::query( + "INSERT INTO torrents (info_hash, completed) VALUES ($1, $2) ON CONFLICT (info_hash) DO UPDATE SET completed = EXCLUDED.completed", + ) + .bind(info_hash.to_hex_string()) + .bind(encoded_downloaded) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(()) + } + } + + async fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { + self.ensure_schema().await?; + + sqlx::query("UPDATE torrents SET completed = completed + 1 WHERE info_hash = $1") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + Ok(()) + } + + async fn load_global_downloads(&self) -> Result, Error> { + self.ensure_schema().await?; + + let row = sqlx::query("SELECT value FROM torrent_aggregate_metrics WHERE metric_name = $1") + .bind(TORRENTS_DOWNLOADS_TOTAL) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + row.map(|row| { + let value: i64 = row.try_get("value").map_err(|err| (err, DRIVER))?; + self.decode_counter_i64(value) + }) + .transpose() + } + + async fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.ensure_schema().await?; + + let encoded_downloaded = self.encode_counter(downloaded)?; + + let insert = sqlx::query( + "INSERT INTO torrent_aggregate_metrics (metric_name, value) VALUES ($1, $2) ON CONFLICT (metric_name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(TORRENTS_DOWNLOADS_TOTAL) + .bind(encoded_downloaded) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(()) + } + } + + async fn increase_global_downloads(&self) -> Result<(), Error> { + self.ensure_schema().await?; + + sqlx::query("UPDATE torrent_aggregate_metrics SET value = value + 1 WHERE metric_name = $1") + .bind(TORRENTS_DOWNLOADS_TOTAL) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + Ok(()) + } +} + +#[async_trait] +impl WhitelistStore for Postgres { + async fn load_whitelist(&self) -> Result, Error> { + self.ensure_schema().await?; + + let rows = sqlx::query("SELECT info_hash FROM whitelist") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + rows.into_iter() + .map(|row| { + let info_hash: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + self.decode_info_hash(info_hash) + }) + .collect() + } + + async fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { + self.ensure_schema().await?; + + let row = sqlx::query("SELECT info_hash FROM whitelist WHERE info_hash = $1") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + row.map(|row| { + let value: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + self.decode_info_hash(value) + }) + .transpose() + } + + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { + self.ensure_schema().await?; + + let insert = sqlx::query("INSERT INTO whitelist (info_hash) VALUES ($1)") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(insert.rows_affected() as usize) + } + } + + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { + self.ensure_schema().await?; + + let deleted = sqlx::query("DELETE FROM whitelist WHERE info_hash = $1") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + let deleted = deleted.rows_affected() as usize; + + if deleted == 1 { + Ok(deleted) + } else { + Err(Error::DeleteFailed { + location: std::panic::Location::caller(), + error_code: deleted, + driver: DRIVER, + }) + } + } +} + +#[async_trait] +impl AuthKeyStore for Postgres { + async fn load_keys(&self) -> Result, Error> { + self.ensure_schema().await?; + + let rows = sqlx::query("SELECT key, valid_until FROM keys") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + rows.into_iter() + .map(|row| { + let key: String = row.try_get("key").map_err(|err| (err, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|err| (err, DRIVER))?; + + Ok(authentication::PeerKey { + key: self.decode_key(key)?, + valid_until: self.decode_valid_until(valid_until)?, + }) + }) + .collect() + } + + async fn get_key_from_keys(&self, key: &Key) -> Result, Error> { + self.ensure_schema().await?; + + let row = sqlx::query("SELECT key, valid_until FROM keys WHERE key = $1") + .bind(key.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + row.map(|row| { + let key: String = row.try_get("key").map_err(|err| (err, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|err| (err, DRIVER))?; + + Ok(authentication::PeerKey { + key: self.decode_key(key)?, + valid_until: self.decode_valid_until(valid_until)?, + }) + }) + .transpose() + } + + async fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result { + self.ensure_schema().await?; + + let valid_until = auth_key + .valid_until + .map(|valid_until| valid_until.as_secs()) + .map(i64::try_from) + .transpose() + .map_err(|err| Error::invalid_query(DRIVER, err))?; + + let insert = sqlx::query("INSERT INTO keys (key, valid_until) VALUES ($1, $2)") + .bind(auth_key.key.to_string()) + .bind(valid_until) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(insert.rows_affected() as usize) + } + } + + async fn remove_key_from_keys(&self, key: &Key) -> Result { + self.ensure_schema().await?; + + let deleted = sqlx::query("DELETE FROM keys WHERE key = $1") + .bind(key.to_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + let deleted = deleted.rows_affected() as usize; + + if deleted == 1 { + Ok(deleted) + } else { + Err(Error::DeleteFailed { + location: std::panic::Location::caller(), + error_code: deleted, + driver: DRIVER, + }) + } + } +} + +#[cfg(test)] +mod tests { + use testcontainers::core::IntoContainerPort; + use testcontainers::runners::AsyncRunner; + use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + use torrust_tracker_configuration::Core; + + use crate::databases::driver::tests::run_tests; + use crate::databases::driver::{build, Driver}; + + #[derive(Debug, Default)] + struct StoppedPostgresContainer {} + + impl StoppedPostgresContainer { + async fn run( + self, + config: &PostgresConfiguration, + ) -> Result> { + let image_name = + std::env::var("TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE").unwrap_or_else(|_| "postgres".to_string()); + let image_tag = std::env::var("TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG").unwrap_or_else(|_| "16".to_string()); + + let container = GenericImage::new(image_name, image_tag) + .with_exposed_port(config.internal_port.tcp()) + .with_env_var("POSTGRES_PASSWORD", config.db_password.clone()) + .with_env_var("POSTGRES_USER", config.db_user.clone()) + .with_env_var("POSTGRES_DB", config.database.clone()) + .start() + .await?; + + Ok(RunningPostgresContainer::new(container, config.internal_port)) + } + } + + struct RunningPostgresContainer { + container: ContainerAsync, + internal_port: u16, + } + + impl RunningPostgresContainer { + fn new(container: ContainerAsync, internal_port: u16) -> Self { + Self { + container, + internal_port, + } + } + + async fn stop(self) { + self.container.stop().await.unwrap(); + } + + async fn get_host(&self) -> url::Host { + self.container.get_host().await.unwrap() + } + + async fn get_host_port_ipv4(&self) -> u16 { + self.container.get_host_port_ipv4(self.internal_port).await.unwrap() + } + } + + impl Default for PostgresConfiguration { + fn default() -> Self { + Self { + internal_port: 5432, + database: "torrust_tracker_test".to_string(), + db_user: "postgres".to_string(), + db_password: "test".to_string(), + } + } + } + + struct PostgresConfiguration { + pub internal_port: u16, + pub database: String, + pub db_user: String, + pub db_password: String, + } + + fn core_configuration(host: &url::Host, port: u16, pg_configuration: &PostgresConfiguration) -> Core { + let mut config = Core::default(); + + config.database.driver = torrust_tracker_configuration::Driver::PostgreSQL; + config.database.path = format!( + "postgresql://{}:{}@{}:{}/{}", + pg_configuration.db_user, pg_configuration.db_password, host, port, pg_configuration.database + ); + + config + } + + fn local_configuration(url: &str) -> Core { + let mut config = Core::default(); + config.database.driver = torrust_tracker_configuration::Driver::PostgreSQL; + config.database.path = url.to_string(); + config + } + + #[tokio::test] + async fn run_postgres_driver_tests() -> Result<(), Box> { + if std::env::var("TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST").is_err() { + println!("Skipping the PostgreSQL driver tests."); + return Ok(()); + } + + if let Ok(database_url) = std::env::var("TORRUST_TRACKER_CORE_POSTGRES_DRIVER_URL") { + let config = local_configuration(&database_url); + let driver = build(&Driver::PostgreSQL, &config.database.path)?; + run_tests(&driver).await; + return Ok(()); + } + + let pg_configuration = PostgresConfiguration::default(); + let stopped_pg_container = StoppedPostgresContainer::default(); + let pg_container = stopped_pg_container.run(&pg_configuration).await.unwrap(); + + let host = pg_container.get_host().await; + let port = pg_container.get_host_port_ipv4().await; + + let config = core_configuration(&host, port, &pg_configuration); + let driver = build(&Driver::PostgreSQL, &config.database.path)?; + + run_tests(&driver).await; + + pg_container.stop().await; + + Ok(()) + } +} diff --git a/packages/tracker-core/src/databases/driver/sqlite.rs b/packages/tracker-core/src/databases/driver/sqlite.rs index d08351aa8..750bd5213 100644 --- a/packages/tracker-core/src/databases/driver/sqlite.rs +++ b/packages/tracker-core/src/databases/driver/sqlite.rs @@ -1,200 +1,197 @@ //! The `SQLite3` database driver. -//! -//! This module provides an implementation of the [`Database`] trait for -//! `SQLite3` using the `r2d2_sqlite` connection pool. It defines the schema for -//! whitelist, torrent metrics, and authentication keys, and provides methods -//! to create and drop tables as well as perform CRUD operations on these -//! persistent objects. -use std::panic::Location; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; +use async_trait::async_trait; use bittorrent_primitives::info_hash::InfoHash; -use r2d2::Pool; -use r2d2_sqlite::rusqlite::params; -use r2d2_sqlite::rusqlite::types::Null; -use r2d2_sqlite::SqliteConnectionManager; +use sqlx::migrate::Migrator; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use sqlx::{ConnectOptions, Row, SqlitePool}; +use tokio::sync::Mutex; use torrust_tracker_primitives::{DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; -use super::{Database, Driver, Error, TORRENTS_DOWNLOADS_TOTAL}; +use super::{Driver, TORRENTS_DOWNLOADS_TOTAL}; use crate::authentication::{self, Key}; +use crate::databases::error::Error; +use crate::databases::{AuthKeyStore, SchemaMigrator, TorrentMetricsStore, WhitelistStore}; const DRIVER: Driver = Driver::Sqlite3; +static MIGRATOR: Migrator = sqlx::migrate!("migrations/sqlite"); -/// `SQLite` driver implementation. -/// -/// This struct encapsulates a connection pool for `SQLite` using the `r2d2_sqlite` -/// connection manager. +/// `SQLite` driver implementation backed by `sqlx`. pub(crate) struct Sqlite { - pool: Pool, + pool: SqlitePool, + schema_ready: AtomicBool, + schema_lock: Mutex<()>, } impl Sqlite { /// Instantiates a new `SQLite3` database driver. /// - /// This function creates a connection manager for the `SQLite` database - /// located at `db_path` and then builds a connection pool using `r2d2`. If - /// the pool cannot be created, an error is returned (wrapped with the - /// appropriate driver information). - /// - /// # Arguments - /// - /// * `db_path` - A string slice representing the file path to the `SQLite` database. - /// /// # Errors /// - /// Returns an [`Error`] if the connection pool cannot be built. + /// Returns an [`Error`] if the database URL cannot be parsed. pub fn new(db_path: &str) -> Result { - let manager = SqliteConnectionManager::file(db_path); - let pool = r2d2::Pool::builder().build(manager).map_err(|e| (e, DRIVER))?; - - Ok(Self { pool }) + let options = SqliteConnectOptions::from_str(db_path) + .map_err(|err| Error::connection_error(DRIVER, err))? + .create_if_missing(true) + .disable_statement_logging(); + + let pool = SqlitePoolOptions::new().connect_lazy_with(options); + + Ok(Self { + pool, + schema_ready: AtomicBool::new(false), + schema_lock: Mutex::new(()), + }) } - fn load_torrent_aggregate_metric(&self, metric_name: &str) -> Result, Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let mut stmt = conn.prepare("SELECT value FROM torrent_aggregate_metrics WHERE metric_name = ?")?; + async fn ensure_schema(&self) -> Result<(), Error> { + if self.schema_ready.load(Ordering::Acquire) { + return Ok(()); + } - let mut rows = stmt.query([metric_name])?; + let _guard = self.schema_lock.lock().await; - let persistent_torrent = rows.next()?; + if self.schema_ready.load(Ordering::Acquire) { + return Ok(()); + } - Ok(persistent_torrent.map(|f| { - let value: i64 = f.get(0).unwrap(); - u32::try_from(value).unwrap() - })) + self.run_migrations().await } - fn save_torrent_aggregate_metric(&self, metric_name: &str, completed: NumberOfDownloads) -> Result<(), Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn run_migrations(&self) -> Result<(), Error> { + MIGRATOR + .run(&self.pool) + .await + .map_err(|err| Error::migration_error(DRIVER, err))?; - let insert = conn.execute( - "INSERT INTO torrent_aggregate_metrics (metric_name, value) VALUES (?1, ?2) ON CONFLICT(metric_name) DO UPDATE SET value = ?2", - [metric_name.to_string(), completed.to_string()], - )?; + self.schema_ready.store(true, Ordering::Release); - if insert == 0 { - Err(Error::InsertFailed { - location: Location::caller(), - driver: DRIVER, - }) - } else { - Ok(()) - } + Ok(()) } -} -impl Database for Sqlite { - /// Refer to [`databases::Database::create_database_tables`](crate::core::databases::Database::create_database_tables). - fn create_database_tables(&self) -> Result<(), Error> { - let create_whitelist_table = " - CREATE TABLE IF NOT EXISTS whitelist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - info_hash TEXT NOT NULL UNIQUE - );" - .to_string(); - - let create_torrents_table = " - CREATE TABLE IF NOT EXISTS torrents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - info_hash TEXT NOT NULL UNIQUE, - completed INTEGER DEFAULT 0 NOT NULL - );" - .to_string(); - - let create_torrent_aggregate_metrics_table = " - CREATE TABLE IF NOT EXISTS torrent_aggregate_metrics ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - metric_name TEXT NOT NULL UNIQUE, - value INTEGER DEFAULT 0 NOT NULL - );" - .to_string(); - - let create_keys_table = " - CREATE TABLE IF NOT EXISTS keys ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - key TEXT NOT NULL UNIQUE, - valid_until INTEGER - );" - .to_string(); - - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - conn.execute(&create_whitelist_table, [])?; - conn.execute(&create_keys_table, [])?; - conn.execute(&create_torrents_table, [])?; - conn.execute(&create_torrent_aggregate_metrics_table, [])?; + fn decode_counter_i64(&self, value: i64) -> Result { + u64::try_from(value).map_err(|err| Error::invalid_query(DRIVER, err)) + } - Ok(()) + fn encode_counter(&self, value: NumberOfDownloads) -> Result { + i64::try_from(value).map_err(|err| Error::invalid_query(DRIVER, err)) } - /// Refer to [`databases::Database::drop_database_tables`](crate::core::databases::Database::drop_database_tables). - fn drop_database_tables(&self) -> Result<(), Error> { - let drop_whitelist_table = " - DROP TABLE whitelist;" - .to_string(); + fn decode_info_hash(&self, value: String) -> Result { + InfoHash::from_str(&value).map_err(|err| { + Error::invalid_query( + DRIVER, + std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{err:?}")), + ) + }) + } - let drop_torrents_table = " - DROP TABLE torrents;" - .to_string(); + fn decode_key(&self, value: String) -> Result { + value.parse::().map_err(|err| Error::invalid_query(DRIVER, err)) + } - let drop_keys_table = " - DROP TABLE keys;" - .to_string(); + fn decode_valid_until(&self, value: Option) -> Result, Error> { + value + .map(|seconds| { + u64::try_from(seconds) + .map(DurationSinceUnixEpoch::from_secs) + .map_err(|err| Error::invalid_query(DRIVER, err)) + }) + .transpose() + } +} - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; +#[async_trait] +impl SchemaMigrator for Sqlite { + async fn create_database_tables(&self) -> Result<(), Error> { + self.run_migrations().await + } - conn.execute(&drop_whitelist_table, []) - .and_then(|_| conn.execute(&drop_torrents_table, [])) - .and_then(|_| conn.execute(&drop_keys_table, []))?; + async fn drop_database_tables(&self) -> Result<(), Error> { + // Tests intentionally use an explicit drop/create cycle to simulate failures. + // Callers that need a working schema after this must invoke `create_database_tables`. + sqlx::query("DROP TABLE IF EXISTS whitelist") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS torrent_aggregate_metrics") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS torrents") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS keys") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + sqlx::query("DROP TABLE IF EXISTS _sqlx_migrations") + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; Ok(()) } +} - /// Refer to [`databases::Database::load_persistent_torrents`](crate::core::databases::Database::load_persistent_torrents). - fn load_all_torrents_downloads(&self) -> Result { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; +#[async_trait] +impl TorrentMetricsStore for Sqlite { + async fn load_all_torrents_downloads(&self) -> Result { + self.ensure_schema().await?; - let mut stmt = conn.prepare("SELECT info_hash, completed FROM torrents")?; + let rows = sqlx::query("SELECT info_hash, completed FROM torrents") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - let torrent_iter = stmt.query_map([], |row| { - let info_hash_string: String = row.get(0)?; - let info_hash = InfoHash::from_str(&info_hash_string).unwrap(); - let completed: u32 = row.get(1)?; - Ok((info_hash, completed)) - })?; + let mut torrents = NumberOfDownloadsBTreeMap::new(); - Ok(torrent_iter.filter_map(std::result::Result::ok).collect()) - } + for row in rows { + let info_hash_string: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + let completed: i64 = row.try_get("completed").map_err(|err| (err, DRIVER))?; - /// Refer to [`databases::Database::load_persistent_torrent`](crate::core::databases::Database::load_persistent_torrent). - fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + torrents.insert(self.decode_info_hash(info_hash_string)?, self.decode_counter_i64(completed)?); + } - let mut stmt = conn.prepare("SELECT completed FROM torrents WHERE info_hash = ?")?; + Ok(torrents) + } - let mut rows = stmt.query([info_hash.to_hex_string()])?; + async fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { + self.ensure_schema().await?; - let persistent_torrent = rows.next()?; + let row = sqlx::query("SELECT completed FROM torrents WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(persistent_torrent.map(|f| { - let completed: i64 = f.get(0).unwrap(); - u32::try_from(completed).unwrap() - })) + row.map(|row| { + let completed: i64 = row.try_get("completed").map_err(|err| (err, DRIVER))?; + self.decode_counter_i64(completed) + }) + .transpose() } - /// Refer to [`databases::Database::save_persistent_torrent`](crate::core::databases::Database::save_persistent_torrent). - fn save_torrent_downloads(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.ensure_schema().await?; + + let encoded_downloaded = self.encode_counter(downloaded)?; - let insert = conn.execute( - "INSERT INTO torrents (info_hash, completed) VALUES (?1, ?2) ON CONFLICT(info_hash) DO UPDATE SET completed = ?2", - [info_hash.to_string(), completed.to_string()], - )?; + let insert = sqlx::query( + "INSERT INTO torrents (info_hash, completed) VALUES (?, ?) ON CONFLICT(info_hash) DO UPDATE SET completed = excluded.completed", + ) + .bind(info_hash.to_hex_string()) + .bind(encoded_downloaded) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - if insert == 0 { + if insert.rows_affected() == 0 { Err(Error::InsertFailed { - location: Location::caller(), + location: std::panic::Location::caller(), driver: DRIVER, }) } else { @@ -202,197 +199,234 @@ impl Database for Sqlite { } } - /// Refer to [`databases::Database::increase_number_of_downloads`](crate::core::databases::Database::increase_number_of_downloads). - fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { + self.ensure_schema().await?; - let _ = conn.execute( - "UPDATE torrents SET completed = completed + 1 WHERE info_hash = ?", - [info_hash.to_string()], - )?; + sqlx::query("UPDATE torrents SET completed = completed + 1 WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; Ok(()) } - /// Refer to [`databases::Database::load_global_number_of_downloads`](crate::core::databases::Database::load_global_number_of_downloads). - fn load_global_downloads(&self) -> Result, Error> { - self.load_torrent_aggregate_metric(TORRENTS_DOWNLOADS_TOTAL) - } - - /// Refer to [`databases::Database::save_global_number_of_downloads`](crate::core::databases::Database::save_global_number_of_downloads). - fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error> { - self.save_torrent_aggregate_metric(TORRENTS_DOWNLOADS_TOTAL, downloaded) - } - - /// Refer to [`databases::Database::increase_global_number_of_downloads`](crate::core::databases::Database::increase_global_number_of_downloads). - fn increase_global_downloads(&self) -> Result<(), Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn load_global_downloads(&self) -> Result, Error> { + self.ensure_schema().await?; - let metric_name = TORRENTS_DOWNLOADS_TOTAL; + let row = sqlx::query("SELECT value FROM torrent_aggregate_metrics WHERE metric_name = ?") + .bind(TORRENTS_DOWNLOADS_TOTAL) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - let _ = conn.execute( - "UPDATE torrent_aggregate_metrics SET value = value + 1 WHERE metric_name = ?", - [metric_name], - )?; - - Ok(()) + row.map(|row| { + let value: i64 = row.try_get("value").map_err(|err| (err, DRIVER))?; + self.decode_counter_i64(value) + }) + .transpose() } - /// Refer to [`databases::Database::load_keys`](crate::core::databases::Database::load_keys). - fn load_keys(&self) -> Result, Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let mut stmt = conn.prepare("SELECT key, valid_until FROM keys")?; - - let keys_iter = stmt.query_map([], |row| { - let key: String = row.get(0)?; - let opt_valid_until: Option = row.get(1)?; + async fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.ensure_schema().await?; - match opt_valid_until { - Some(valid_until) => Ok(authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: Some(DurationSinceUnixEpoch::from_secs(valid_until.unsigned_abs())), - }), - None => Ok(authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: None, - }), - } - })?; + let encoded_downloaded = self.encode_counter(downloaded)?; - let keys: Vec = keys_iter.filter_map(std::result::Result::ok).collect(); + let insert = sqlx::query( + "INSERT INTO torrent_aggregate_metrics (metric_name, value) VALUES (?, ?) ON CONFLICT(metric_name) DO UPDATE SET value = excluded.value", + ) + .bind(TORRENTS_DOWNLOADS_TOTAL) + .bind(encoded_downloaded) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(keys) + if insert.rows_affected() == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + Ok(()) + } } - /// Refer to [`databases::Database::load_whitelist`](crate::core::databases::Database::load_whitelist). - fn load_whitelist(&self) -> Result, Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn increase_global_downloads(&self) -> Result<(), Error> { + self.ensure_schema().await?; - let mut stmt = conn.prepare("SELECT info_hash FROM whitelist")?; + sqlx::query("UPDATE torrent_aggregate_metrics SET value = value + 1 WHERE metric_name = ?") + .bind(TORRENTS_DOWNLOADS_TOTAL) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - let info_hash_iter = stmt.query_map([], |row| { - let info_hash: String = row.get(0)?; + Ok(()) + } +} - Ok(InfoHash::from_str(&info_hash).unwrap()) - })?; +#[async_trait] +impl WhitelistStore for Sqlite { + async fn load_whitelist(&self) -> Result, Error> { + self.ensure_schema().await?; - let info_hashes: Vec = info_hash_iter.filter_map(std::result::Result::ok).collect(); + let rows = sqlx::query("SELECT info_hash FROM whitelist") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(info_hashes) + rows.into_iter() + .map(|row| { + let info_hash: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + self.decode_info_hash(info_hash) + }) + .collect() } - /// Refer to [`databases::Database::get_info_hash_from_whitelist`](crate::core::databases::Database::get_info_hash_from_whitelist). - fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let mut stmt = conn.prepare("SELECT info_hash FROM whitelist WHERE info_hash = ?")?; - - let mut rows = stmt.query([info_hash.to_hex_string()])?; + async fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { + self.ensure_schema().await?; - let query = rows.next()?; + let row = sqlx::query("SELECT info_hash FROM whitelist WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(query.map(|f| InfoHash::from_str(&f.get_unwrap::<_, String>(0)).unwrap())) + row.map(|row| { + let value: String = row.try_get("info_hash").map_err(|err| (err, DRIVER))?; + self.decode_info_hash(value) + }) + .transpose() } - /// Refer to [`databases::Database::add_info_hash_to_whitelist`](crate::core::databases::Database::add_info_hash_to_whitelist). - fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { + self.ensure_schema().await?; - let insert = conn.execute("INSERT INTO whitelist (info_hash) VALUES (?)", [info_hash.to_string()])?; + let insert = sqlx::query("INSERT INTO whitelist (info_hash) VALUES (?)") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - if insert == 0 { + if insert.rows_affected() == 0 { Err(Error::InsertFailed { - location: Location::caller(), + location: std::panic::Location::caller(), driver: DRIVER, }) } else { - Ok(insert) + Ok(insert.rows_affected() as usize) } } - /// Refer to [`databases::Database::remove_info_hash_from_whitelist`](crate::core::databases::Database::remove_info_hash_from_whitelist). - fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { + self.ensure_schema().await?; - let deleted = conn.execute("DELETE FROM whitelist WHERE info_hash = ?", [info_hash.to_string()])?; + let deleted = sqlx::query("DELETE FROM whitelist WHERE info_hash = ?") + .bind(info_hash.to_hex_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + let deleted = deleted.rows_affected() as usize; if deleted == 1 { - // should only remove a single record. Ok(deleted) } else { Err(Error::DeleteFailed { - location: Location::caller(), + location: std::panic::Location::caller(), error_code: deleted, driver: DRIVER, }) } } +} - /// Refer to [`databases::Database::get_key_from_keys`](crate::core::databases::Database::get_key_from_keys). - fn get_key_from_keys(&self, key: &Key) -> Result, Error> { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let mut stmt = conn.prepare("SELECT key, valid_until FROM keys WHERE key = ?")?; +#[async_trait] +impl AuthKeyStore for Sqlite { + async fn load_keys(&self) -> Result, Error> { + self.ensure_schema().await?; + + let rows = sqlx::query("SELECT key, valid_until FROM keys") + .fetch_all(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + rows.into_iter() + .map(|row| { + let key: String = row.try_get("key").map_err(|err| (err, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|err| (err, DRIVER))?; + + Ok(authentication::PeerKey { + key: self.decode_key(key)?, + valid_until: self.decode_valid_until(valid_until)?, + }) + }) + .collect() + } - let mut rows = stmt.query([key.to_string()])?; + async fn get_key_from_keys(&self, key: &Key) -> Result, Error> { + self.ensure_schema().await?; - let key = rows.next()?; + let row = sqlx::query("SELECT key, valid_until FROM keys WHERE key = ?") + .bind(key.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - Ok(key.map(|f| { - let valid_until: Option = f.get(1).unwrap(); - let key: String = f.get(0).unwrap(); + row.map(|row| { + let key: String = row.try_get("key").map_err(|err| (err, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|err| (err, DRIVER))?; - match valid_until { - Some(valid_until) => authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: Some(DurationSinceUnixEpoch::from_secs(valid_until.unsigned_abs())), - }, - None => authentication::PeerKey { - key: key.parse::().unwrap(), - valid_until: None, - }, - } - })) + Ok(authentication::PeerKey { + key: self.decode_key(key)?, + valid_until: self.decode_valid_until(valid_until)?, + }) + }) + .transpose() } - /// Refer to [`databases::Database::add_key_to_keys`](crate::core::databases::Database::add_key_to_keys). - fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; - - let insert = match auth_key.valid_until { - Some(valid_until) => conn.execute( - "INSERT INTO keys (key, valid_until) VALUES (?1, ?2)", - [auth_key.key.to_string(), valid_until.as_secs().to_string()], - )?, - None => conn.execute( - "INSERT INTO keys (key, valid_until) VALUES (?1, ?2)", - params![auth_key.key.to_string(), Null], - )?, - }; - - if insert == 0 { + async fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result { + self.ensure_schema().await?; + + let valid_until = auth_key + .valid_until + .map(|valid_until| valid_until.as_secs()) + .map(i64::try_from) + .transpose() + .map_err(|err| Error::invalid_query(DRIVER, err))?; + + let insert = sqlx::query("INSERT INTO keys (key, valid_until) VALUES (?, ?)") + .bind(auth_key.key.to_string()) + .bind(valid_until) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; + + if insert.rows_affected() == 0 { Err(Error::InsertFailed { - location: Location::caller(), + location: std::panic::Location::caller(), driver: DRIVER, }) } else { - Ok(insert) + Ok(insert.rows_affected() as usize) } } - /// Refer to [`databases::Database::remove_key_from_keys`](crate::core::databases::Database::remove_key_from_keys). - fn remove_key_from_keys(&self, key: &Key) -> Result { - let conn = self.pool.get().map_err(|e| (e, DRIVER))?; + async fn remove_key_from_keys(&self, key: &Key) -> Result { + self.ensure_schema().await?; + + let deleted = sqlx::query("DELETE FROM keys WHERE key = ?") + .bind(key.to_string()) + .execute(&self.pool) + .await + .map_err(|err| (err, DRIVER))?; - let deleted = conn.execute("DELETE FROM keys WHERE key = ?", [key.to_string()])?; + let deleted = deleted.rows_affected() as usize; if deleted == 1 { - // should only remove a single record. Ok(deleted) } else { Err(Error::DeleteFailed { - location: Location::caller(), + location: std::panic::Location::caller(), error_code: deleted, driver: DRIVER, }) @@ -402,15 +436,11 @@ impl Database for Sqlite { #[cfg(test)] mod tests { - - use std::sync::Arc; - use torrust_tracker_configuration::Core; use torrust_tracker_test_helpers::configuration::ephemeral_sqlite_database; - use crate::databases::driver::sqlite::Sqlite; use crate::databases::driver::tests::run_tests; - use crate::databases::Database; + use crate::databases::driver::{build, Driver}; fn ephemeral_configuration() -> Core { let mut config = Core::default(); @@ -419,16 +449,10 @@ mod tests { config } - fn initialize_driver(config: &Core) -> Arc> { - let driver: Arc> = Arc::new(Box::new(Sqlite::new(&config.database.path).unwrap())); - driver - } - #[tokio::test] async fn run_sqlite_driver_tests() -> Result<(), Box> { let config = ephemeral_configuration(); - - let driver = initialize_driver(&config); + let driver = build(&Driver::Sqlite3, &config.database.path)?; run_tests(&driver).await; diff --git a/packages/tracker-core/src/databases/error.rs b/packages/tracker-core/src/databases/error.rs index 2df2cb277..aa914b46b 100644 --- a/packages/tracker-core/src/databases/error.rs +++ b/packages/tracker-core/src/databases/error.rs @@ -1,39 +1,24 @@ //! Database errors. -//! -//! This module defines the [`Error`] enum used to represent errors that occur -//! during database operations. These errors encapsulate issues such as missing -//! query results, malformed queries, connection failures, and connection pool -//! creation errors. Each error variant includes contextual information such as -//! the associated database driver and, when applicable, the source error. -//! -//! External errors from database libraries (e.g., `rusqlite`, `mysql`) are -//! converted into this error type using the provided `From` implementations. use std::panic::Location; use std::sync::Arc; -use r2d2_mysql::mysql::UrlError; -use torrust_tracker_located_error::{DynError, Located, LocatedError}; +use sqlx::Error as SqlxError; +use torrust_tracker_located_error::{DynError, LocatedError}; use super::driver::Driver; /// Database error type that encapsulates various failures encountered during -/// database operations. +/// persistence operations. #[derive(thiserror::Error, Debug, Clone)] pub enum Error { /// Indicates that a query unexpectedly returned no rows. - /// - /// This error variant is used when a query that is expected to return a - /// result does not. #[error("The {driver} query unexpectedly returned nothing: {source}")] QueryReturnedNoRows { source: LocatedError<'static, dyn std::error::Error + Send + Sync>, driver: Driver, }, - /// Indicates that the query was malformed. - /// - /// This error variant is used when the SQL query itself is invalid or - /// improperly formatted. + /// Indicates that the query was malformed or its returned data could not be decoded. #[error("The {driver} query was malformed: {source}")] InvalidQuery { source: LocatedError<'static, dyn std::error::Error + Send + Sync>, @@ -41,8 +26,6 @@ pub enum Error { }, /// Indicates a failure to insert a record into the database. - /// - /// This error is raised when an insertion operation fails. #[error("Unable to insert record into {driver} database, {location}")] InsertFailed { location: &'static Location<'static>, @@ -50,8 +33,6 @@ pub enum Error { }, /// Indicates a failure to update a record into the database. - /// - /// This error is raised when an insertion operation fails. #[error("Unable to update record into {driver} database, {location}")] UpdateFailed { location: &'static Location<'static>, @@ -59,9 +40,6 @@ pub enum Error { }, /// Indicates a failure to delete a record from the database. - /// - /// This error includes an error code that may be returned by the database - /// driver. #[error("Failed to remove record from {driver} database, error-code: {error_code}, {location}")] DeleteFailed { location: &'static Location<'static>, @@ -70,103 +48,177 @@ pub enum Error { }, /// Indicates a failure to connect to the database. - /// - /// This error variant wraps connection-related errors, such as those caused by an invalid URL. #[error("Failed to connect to {driver} database: {source}")] ConnectionError { - source: LocatedError<'static, UrlError>, + source: LocatedError<'static, dyn std::error::Error + Send + Sync>, driver: Driver, }, - /// Indicates a failure to create a connection pool. - /// - /// This error variant is used when the connection pool creation (using r2d2) fails. - #[error("Failed to create r2d2 {driver} connection pool: {source}")] - ConnectionPool { - source: LocatedError<'static, r2d2::Error>, + /// Indicates a failure while running migrations. + #[error("Failed to run {driver} database migrations: {source}")] + MigrationError { + source: LocatedError<'static, dyn std::error::Error + Send + Sync>, driver: Driver, }, } -impl From for Error { +impl Error { + /// Builds an invalid query error. + #[track_caller] + pub fn invalid_query(driver: Driver, err: E) -> Self + where + E: std::error::Error + Send + Sync + 'static, + { + Self::InvalidQuery { + source: (Arc::new(err) as DynError).into(), + driver, + } + } + + /// Builds a query returned no rows error. + #[track_caller] + pub fn query_returned_no_rows(driver: Driver, err: E) -> Self + where + E: std::error::Error + Send + Sync + 'static, + { + Self::QueryReturnedNoRows { + source: (Arc::new(err) as DynError).into(), + driver, + } + } + + /// Builds a connection error. + #[track_caller] + pub fn connection_error(driver: Driver, err: E) -> Self + where + E: std::error::Error + Send + Sync + 'static, + { + Self::ConnectionError { + source: (Arc::new(err) as DynError).into(), + driver, + } + } + + /// Builds a migration error. + #[track_caller] + pub fn migration_error(driver: Driver, err: E) -> Self + where + E: std::error::Error + Send + Sync + 'static, + { + Self::MigrationError { + source: (Arc::new(err) as DynError).into(), + driver, + } + } +} + +impl From<(SqlxError, Driver)> for Error { #[track_caller] - fn from(err: r2d2_sqlite::rusqlite::Error) -> Self { + fn from(value: (SqlxError, Driver)) -> Self { + let (err, driver) = value; + match err { - r2d2_sqlite::rusqlite::Error::QueryReturnedNoRows => Error::QueryReturnedNoRows { + SqlxError::RowNotFound => Self::QueryReturnedNoRows { + source: (Arc::new(SqlxError::RowNotFound) as DynError).into(), + driver, + }, + SqlxError::Io(_) + | SqlxError::Tls(_) + | SqlxError::PoolTimedOut + | SqlxError::PoolClosed + | SqlxError::WorkerCrashed + | SqlxError::Configuration(_) => Self::ConnectionError { source: (Arc::new(err) as DynError).into(), - driver: Driver::Sqlite3, + driver, }, - _ => Error::InvalidQuery { + _ => Self::InvalidQuery { source: (Arc::new(err) as DynError).into(), - driver: Driver::Sqlite3, + driver, }, } } } -impl From for Error { +impl From<(DynError, Driver)> for Error { #[track_caller] - fn from(err: r2d2_mysql::mysql::Error) -> Self { - let e: DynError = Arc::new(err); - Error::InvalidQuery { - source: e.into(), - driver: Driver::MySQL, - } - } -} + fn from(value: (DynError, Driver)) -> Self { + let (err, driver) = value; -impl From for Error { - #[track_caller] - fn from(err: UrlError) -> Self { Self::ConnectionError { - source: Located(err).into(), - driver: Driver::MySQL, + source: err.into(), + driver, } } } -impl From<(r2d2::Error, Driver)> for Error { +impl From<(LocatedError<'static, dyn std::error::Error + Send + Sync>, Driver)> for Error { #[track_caller] - fn from(e: (r2d2::Error, Driver)) -> Self { - let (err, driver) = e; - Self::ConnectionPool { - source: Located(err).into(), - driver, - } + fn from(value: (LocatedError<'static, dyn std::error::Error + Send + Sync>, Driver)) -> Self { + let (source, driver) = value; + + Self::ConnectionError { source, driver } } } #[cfg(test)] mod tests { - use r2d2_mysql::mysql; + use std::io; + use std::sync::Arc; + + use torrust_tracker_located_error::DynError; - use crate::databases::error::Error; + use super::Error; + use crate::databases::driver::Driver; #[test] - fn it_should_build_a_database_error_from_a_rusqlite_error() { - let err: Error = r2d2_sqlite::rusqlite::Error::InvalidQuery.into(); + fn it_should_build_a_database_error_from_a_sqlx_row_not_found_error() { + let err: Error = (sqlx::Error::RowNotFound, Driver::Sqlite3).into(); - assert!(matches!(err, Error::InvalidQuery { .. })); + assert!(matches!(err, Error::QueryReturnedNoRows { .. })); } #[test] - fn it_should_build_an_specific_database_error_from_a_no_rows_returned_rusqlite_error() { - let err: Error = r2d2_sqlite::rusqlite::Error::QueryReturnedNoRows.into(); + fn it_should_build_a_database_error_from_a_sqlx_io_error() { + let err: Error = ( + sqlx::Error::Io(io::Error::from(io::ErrorKind::ConnectionRefused)), + Driver::MySQL, + ) + .into(); - assert!(matches!(err, Error::QueryReturnedNoRows { .. })); + assert!(matches!(err, Error::ConnectionError { .. })); + } + + #[test] + fn it_should_build_a_database_error_from_a_dyn_connection_error() { + let err: Error = ( + (Arc::new(io::Error::from(io::ErrorKind::TimedOut)) as DynError), + Driver::PostgreSQL, + ) + .into(); + + assert!(matches!(err, Error::ConnectionError { .. })); + } + + #[test] + fn it_should_build_a_migration_error() { + let err = Error::migration_error(Driver::Sqlite3, io::Error::from(io::ErrorKind::InvalidData)); + + assert!(matches!(err, Error::MigrationError { .. })); } #[test] - fn it_should_build_a_database_error_from_a_mysql_error() { - let url_err = mysql::error::UrlError::BadUrl; - let err: Error = r2d2_mysql::mysql::Error::UrlError(url_err).into(); + fn it_should_build_an_invalid_query_error() { + let err = Error::invalid_query(Driver::MySQL, io::Error::from(io::ErrorKind::InvalidData)); assert!(matches!(err, Error::InvalidQuery { .. })); } #[test] - fn it_should_build_a_database_error_from_a_mysql_url_error() { - let err: Error = mysql::error::UrlError::BadUrl.into(); + fn it_should_build_a_located_connection_error() { + let err = Error::ConnectionError { + source: (Arc::new(io::Error::from(io::ErrorKind::TimedOut)) as DynError).into(), + driver: Driver::Sqlite3, + }; assert!(matches!(err, Error::ConnectionError { .. })); } diff --git a/packages/tracker-core/src/databases/mod.rs b/packages/tracker-core/src/databases/mod.rs index c9d89769a..a49d2d8a4 100644 --- a/packages/tracker-core/src/databases/mod.rs +++ b/packages/tracker-core/src/databases/mod.rs @@ -1,55 +1,27 @@ //! The persistence module. //! -//! Persistence is currently implemented using a single [`Database`] trait. +//! Persistence is implemented through dedicated store traits per domain +//! context. Each backend is responsible for: //! -//! There are two implementations of the trait (two drivers): +//! - running schema migrations +//! - persisting torrent metrics +//! - persisting the whitelist +//! - persisting authentication keys //! -//! - **`MySQL`** -//! - **`Sqlite`** -//! -//! > **NOTICE**: There are no database migrations at this time. If schema -//! > changes occur, either migration functionality will be implemented or a -//! > script will be provided to migrate to the new schema. -//! -//! The persistent objects handled by this module include: -//! -//! - **Torrent metrics**: Metrics such as the number of completed downloads for -//! each torrent. -//! - **Torrent whitelist**: A list of torrents (by infohash) that are allowed. -//! - **Authentication keys**: Expiring authentication keys used to secure -//! access to private trackers. -//! -//! # Torrent Metrics -//! -//! | Field | Sample data | Description | -//! |-------------|--------------------------------------------|-----------------------------------------------------------------------------| -//! | `id` | 1 | Auto-increment id | -//! | `info_hash` | `c1277613db1d28709b034a017ab2cae4be07ae10` | `BitTorrent` infohash V1 | -//! | `completed` | 20 | The number of peers that have completed downloading the associated torrent. | -//! -//! > **NOTICE**: The peer list for a torrent is not persisted. Because peers re-announce at -//! > intervals, the peer list is regenerated periodically. -//! -//! # Torrent Whitelist -//! -//! | Field | Sample data | Description | -//! |-------------|--------------------------------------------|--------------------------------| -//! | `id` | 1 | Auto-increment id | -//! | `info_hash` | `c1277613db1d28709b034a017ab2cae4be07ae10` | `BitTorrent` infohash V1 | +//! The supported drivers are: //! -//! # Authentication Keys -//! -//! | Field | Sample data | Description | -//! |---------------|------------------------------------|--------------------------------------| -//! | `id` | 1 | Auto-increment id | -//! | `key` | `IrweYtVuQPGbG9Jzx1DihcPmJGGpVy82` | Authentication token (32 chars) | -//! | `valid_until` | 1672419840 | Timestamp indicating expiration time | +//! - **`MySQL`** +//! - **`PostgreSQL`** +//! - **`Sqlite3`** //! -//! > **NOTICE**: All authentication keys must have an expiration date. +//! The schema source of truth is the SQL files in `migrations/`. pub mod driver; pub mod error; pub mod setup; +use std::sync::Arc; + +use async_trait::async_trait; use bittorrent_primitives::info_hash::InfoHash; use mockall::automock; use torrust_tracker_primitives::{NumberOfDownloads, NumberOfDownloadsBTreeMap}; @@ -57,216 +29,204 @@ use torrust_tracker_primitives::{NumberOfDownloads, NumberOfDownloadsBTreeMap}; use self::error::Error; use crate::authentication::{self, Key}; -/// The persistence trait. -/// -/// This trait defines all the methods required to interact with the database, -/// including creating and dropping schema tables, and CRUD operations for -/// torrent metrics, whitelists, and authentication keys. Implementations of -/// this trait must ensure that operations are safe, consistent, and report -/// errors using the [`Error`] type. +/// Shared persistence handles grouped by context. +#[derive(Clone)] +pub struct Persistence { + schema_migrator: Arc, + torrent_metrics_store: Arc, + whitelist_store: Arc, + auth_key_store: Arc, +} + +impl Persistence { + /// Builds a new set of persistence handles. + #[must_use] + pub fn new( + schema_migrator: Arc, + torrent_metrics_store: Arc, + whitelist_store: Arc, + auth_key_store: Arc, + ) -> Self { + Self { + schema_migrator, + torrent_metrics_store, + whitelist_store, + auth_key_store, + } + } + + /// Returns the schema migrator handle. + #[must_use] + pub fn schema_migrator(&self) -> Arc { + self.schema_migrator.clone() + } + + /// Returns the torrent metrics store handle. + #[must_use] + pub fn torrent_metrics_store(&self) -> Arc { + self.torrent_metrics_store.clone() + } + + /// Returns the whitelist store handle. + #[must_use] + pub fn whitelist_store(&self) -> Arc { + self.whitelist_store.clone() + } + + /// Returns the authentication key store handle. + #[must_use] + pub fn auth_key_store(&self) -> Arc { + self.auth_key_store.clone() + } +} + +/// Schema migration operations. #[automock] -pub trait Database: Sync + Send { - /// Creates the necessary database tables. - /// - /// The SQL queries for table creation are hardcoded in the trait implementation. - /// - /// # Context: Schema +#[async_trait] +pub trait SchemaMigrator: Sync + Send { + /// Creates or migrates the database schema. /// /// # Errors /// - /// Returns an [`Error`] if the tables cannot be created. - fn create_database_tables(&self) -> Result<(), Error>; + /// Returns an [`Error`] if the schema cannot be created or migrated. + async fn create_database_tables(&self) -> Result<(), Error>; - /// Drops the database tables. - /// - /// This operation removes the persistent schema. - /// - /// # Context: Schema + /// Drops all persistence tables. /// /// # Errors /// - /// Returns an [`Error`] if the tables cannot be dropped. - fn drop_database_tables(&self) -> Result<(), Error>; - - // Torrent Metrics + /// Returns an [`Error`] if any table cannot be dropped. + async fn drop_database_tables(&self) -> Result<(), Error>; +} - /// Loads torrent metrics data from the database for all torrents. - /// - /// This function returns the persistent torrent metrics as a collection of - /// tuples, where each tuple contains an [`InfoHash`] and the `downloaded` - /// counter (i.e. the number of times the torrent has been downloaded). - /// - /// # Context: Torrent Metrics +/// Torrent metrics persistence. +#[automock] +#[async_trait] +pub trait TorrentMetricsStore: Sync + Send { + /// Loads torrent download counters for all torrents. /// /// # Errors /// - /// Returns an [`Error`] if the metrics cannot be loaded. - fn load_all_torrents_downloads(&self) -> Result; + /// Returns an [`Error`] if the data cannot be loaded. + async fn load_all_torrents_downloads(&self) -> Result; - /// Loads torrent metrics data from the database for one torrent. - /// - /// # Context: Torrent Metrics + /// Loads torrent download counters for one torrent. /// /// # Errors /// - /// Returns an [`Error`] if the metrics cannot be loaded. - fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error>; + /// Returns an [`Error`] if the data cannot be loaded. + async fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error>; - /// Saves torrent metrics data into the database. - /// - /// # Arguments - /// - /// * `info_hash` - A reference to the torrent's info hash. - /// * `downloaded` - The number of times the torrent has been downloaded. - /// - /// # Context: Torrent Metrics + /// Saves torrent download counters. /// /// # Errors /// - /// Returns an [`Error`] if the metrics cannot be saved. - fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: u32) -> Result<(), Error>; + /// Returns an [`Error`] if the data cannot be saved. + async fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: NumberOfDownloads) -> Result<(), Error>; - /// Increases the number of downloads for a given torrent. - /// - /// It does not create a new entry if the torrent is not found and it does - /// not return an error. - /// - /// # Context: Torrent Metrics - /// - /// # Arguments + /// Increases the download counter for a torrent. /// - /// * `info_hash` - A reference to the torrent's info hash. + /// Implementations may treat a missing row as a no-op. The repository + /// layer is responsible for inserting the counter when it does not exist. /// /// # Errors /// - /// Returns an [`Error`] if the query failed. - fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error>; + /// Returns an [`Error`] if the update fails. + async fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error>; - /// Loads the total number of downloads for all torrents from the database. - /// - /// # Context: Torrent Metrics + /// Loads the global number of downloads. /// /// # Errors /// - /// Returns an [`Error`] if the total downloads cannot be loaded. - fn load_global_downloads(&self) -> Result, Error>; + /// Returns an [`Error`] if the data cannot be loaded. + async fn load_global_downloads(&self) -> Result, Error>; - /// Saves the total number of downloads for all torrents into the database. - /// - /// # Context: Torrent Metrics - /// - /// # Arguments - /// - /// * `info_hash` - A reference to the torrent's info hash. - /// * `downloaded` - The number of times the torrent has been downloaded. + /// Saves the global number of downloads. /// /// # Errors /// - /// Returns an [`Error`] if the total downloads cannot be saved. - fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error>; + /// Returns an [`Error`] if the data cannot be saved. + async fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error>; - /// Increases the total number of downloads for all torrents. - /// - /// # Context: Torrent Metrics + /// Increases the global number of downloads. /// /// # Errors /// - /// Returns an [`Error`] if the query failed. - fn increase_global_downloads(&self) -> Result<(), Error>; - - // Whitelist + /// Returns an [`Error`] if the update fails. + async fn increase_global_downloads(&self) -> Result<(), Error>; +} - /// Loads the whitelisted torrents from the database. - /// - /// # Context: Whitelist +/// Torrent whitelist persistence. +#[automock] +#[async_trait] +pub trait WhitelistStore: Sync + Send { + /// Loads all whitelisted torrents. /// /// # Errors /// /// Returns an [`Error`] if the whitelist cannot be loaded. - fn load_whitelist(&self) -> Result, Error>; + async fn load_whitelist(&self) -> Result, Error>; - /// Retrieves a whitelisted torrent from the database. - /// - /// Returns `Some(InfoHash)` if the torrent is in the whitelist, or `None` - /// otherwise. - /// - /// # Context: Whitelist + /// Retrieves a whitelisted torrent. /// /// # Errors /// /// Returns an [`Error`] if the whitelist cannot be queried. - fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error>; + async fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error>; /// Adds a torrent to the whitelist. /// - /// # Context: Whitelist - /// /// # Errors /// - /// Returns an [`Error`] if the torrent cannot be added to the whitelist. - fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result; + /// Returns an [`Error`] if the torrent cannot be added. + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result; - /// Checks whether a torrent is whitelisted. - /// - /// This default implementation returns `true` if the infohash is included - /// in the whitelist, or `false` otherwise. - /// - /// # Context: Whitelist + /// Removes a torrent from the whitelist. /// /// # Errors /// - /// Returns an [`Error`] if the whitelist cannot be queried. - fn is_info_hash_whitelisted(&self, info_hash: InfoHash) -> Result { - Ok(self.get_info_hash_from_whitelist(info_hash)?.is_some()) - } + /// Returns an [`Error`] if the torrent cannot be removed. + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result; - /// Removes a torrent from the whitelist. - /// - /// # Context: Whitelist + /// Checks whether a torrent is whitelisted. /// /// # Errors /// - /// Returns an [`Error`] if the torrent cannot be removed from the whitelist. - fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result; - - // Authentication keys + /// Returns an [`Error`] if the whitelist cannot be queried. + async fn is_info_hash_whitelisted(&self, info_hash: InfoHash) -> Result { + Ok(self.get_info_hash_from_whitelist(info_hash).await?.is_some()) + } +} - /// Loads all authentication keys from the database. - /// - /// # Context: Authentication Keys +/// Authentication key persistence. +#[automock] +#[async_trait] +pub trait AuthKeyStore: Sync + Send { + /// Loads all authentication keys. /// /// # Errors /// /// Returns an [`Error`] if the keys cannot be loaded. - fn load_keys(&self) -> Result, Error>; + async fn load_keys(&self) -> Result, Error>; - /// Retrieves a specific authentication key from the database. - /// - /// Returns `Some(PeerKey)` if a key corresponding to the provided [`Key`] - /// exists, or `None` otherwise. - /// - /// # Context: Authentication Keys + /// Retrieves a specific authentication key. /// /// # Errors /// /// Returns an [`Error`] if the key cannot be queried. - fn get_key_from_keys(&self, key: &Key) -> Result, Error>; + async fn get_key_from_keys(&self, key: &Key) -> Result, Error>; - /// Adds an authentication key to the database. - /// - /// # Context: Authentication Keys + /// Adds an authentication key. /// /// # Errors /// /// Returns an [`Error`] if the key cannot be saved. - fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result; + async fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result; - /// Removes an authentication key from the database. - /// - /// # Context: Authentication Keys + /// Removes an authentication key. /// /// # Errors /// /// Returns an [`Error`] if the key cannot be removed. - fn remove_key_from_keys(&self, key: &Key) -> Result; + async fn remove_key_from_keys(&self, key: &Key) -> Result; } diff --git a/packages/tracker-core/src/databases/setup.rs b/packages/tracker-core/src/databases/setup.rs index 6ba9f2a64..40f2a4d78 100644 --- a/packages/tracker-core/src/databases/setup.rs +++ b/packages/tracker-core/src/databases/setup.rs @@ -1,51 +1,27 @@ //! This module provides functionality for setting up databases. -use std::sync::Arc; - use torrust_tracker_configuration::Core; use super::driver::{self, Driver}; -use super::Database; +use super::Persistence; -/// Initializes and returns a database instance based on the provided configuration. -/// -/// This function creates a new database instance according to the settings -/// defined in the [`Core`] configuration. It selects the appropriate driver -/// (either `Sqlite3` or `MySQL`) as specified in `config.database.driver` and -/// attempts to build the database connection using the path defined in -/// `config.database.path`. +/// Initializes and returns persistence handles based on the provided configuration. /// -/// The resulting database instance is wrapped in a shared pointer (`Arc`) to a -/// boxed trait object, allowing safe sharing of the database connection across -/// multiple threads. +/// This function creates a new persistence backend according to the settings +/// defined in the [`Core`] configuration. The returned value groups the schema +/// migrator and the per-context stores. /// /// # Panics /// -/// This function will panic if the database cannot be initialized (i.e., if the -/// driver fails to build the connection). This is enforced by the use of -/// [`expect`](std::result::Result::expect) in the implementation. -/// -/// # Example -/// -/// ```rust,no_run -/// use torrust_tracker_configuration::Core; -/// use bittorrent_tracker_core::databases::setup::initialize_database; -/// -/// // Create a default configuration (ensure it is properly set up for your environment) -/// let config = Core::default(); -/// -/// // Initialize the database; this will panic if initialization fails. -/// let database = initialize_database(&config); -/// -/// // The returned database instance can now be used for persistence operations. -/// ``` +/// This function will panic if the database backend cannot be initialized. #[must_use] -pub fn initialize_database(config: &Core) -> Arc> { +pub fn initialize_database(config: &Core) -> Persistence { let driver = match config.database.driver { torrust_tracker_configuration::Driver::Sqlite3 => Driver::Sqlite3, torrust_tracker_configuration::Driver::MySQL => Driver::MySQL, + torrust_tracker_configuration::Driver::PostgreSQL => Driver::PostgreSQL, }; - Arc::new(driver::build(&driver, &config.database.path).expect("Database driver build failed.")) + driver::build(&driver, &config.database.path).expect("Database driver build failed.") } #[cfg(test)] @@ -53,8 +29,8 @@ mod tests { use super::initialize_database; use crate::test_helpers::tests::ephemeral_configuration; - #[test] - fn it_should_initialize_the_sqlite_database() { + #[tokio::test] + async fn it_should_initialize_the_sqlite_database() { let config = ephemeral_configuration(); let _database = initialize_database(&config); } diff --git a/packages/tracker-core/src/statistics/event/handler.rs b/packages/tracker-core/src/statistics/event/handler.rs index 9a5182f25..afcff4e82 100644 --- a/packages/tracker-core/src/statistics/event/handler.rs +++ b/packages/tracker-core/src/statistics/event/handler.rs @@ -53,7 +53,10 @@ pub async fn handle_event( if persistent_torrent_completed_stat { // Increment the number of downloads for the torrent in the database - match db_downloads_metric_repository.increase_downloads_for_torrent(&info_hash) { + match db_downloads_metric_repository + .increase_downloads_for_torrent(&info_hash) + .await + { Ok(()) => { tracing::debug!(info_hash = ?info_hash, "Number of torrent downloads increased"); } @@ -63,7 +66,7 @@ pub async fn handle_event( } // Increment the global number of downloads (for all torrents) in the database - match db_downloads_metric_repository.increase_global_downloads() { + match db_downloads_metric_repository.increase_global_downloads().await { Ok(()) => { tracing::debug!("Global number of downloads increased"); } diff --git a/packages/tracker-core/src/statistics/persisted/downloads.rs b/packages/tracker-core/src/statistics/persisted/downloads.rs index 6248bdc73..50b1393fa 100644 --- a/packages/tracker-core/src/statistics/persisted/downloads.rs +++ b/packages/tracker-core/src/statistics/persisted/downloads.rs @@ -5,7 +5,7 @@ use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_primitives::{NumberOfDownloads, NumberOfDownloadsBTreeMap}; use crate::databases::error::Error; -use crate::databases::Database; +use crate::databases::TorrentMetricsStore; /// It persists torrent metrics in a database. /// @@ -22,10 +22,8 @@ use crate::databases::Database; pub struct DatabaseDownloadsMetricRepository { /// A shared reference to the database driver implementation. /// - /// The driver must implement the [`Database`] trait. This allows for - /// different underlying implementations (e.g., `SQLite3` or `MySQL`) to be - /// used interchangeably. - database: Arc>, + /// The store must implement the [`TorrentMetricsStore`] trait. + database: Arc, } impl DatabaseDownloadsMetricRepository { @@ -33,18 +31,15 @@ impl DatabaseDownloadsMetricRepository { /// /// # Arguments /// - /// * `database` - A shared reference to a boxed database driver - /// implementing the [`Database`] trait. + /// * `database` - A shared reference to a torrent metrics store. /// /// # Returns /// /// A new `DatabasePersistentTorrentRepository` instance with a cloned /// reference to the provided database. #[must_use] - pub fn new(database: &Arc>) -> DatabaseDownloadsMetricRepository { - Self { - database: database.clone(), - } + pub fn new(database: Arc) -> DatabaseDownloadsMetricRepository { + Self { database } } // Single Torrent Metrics @@ -60,12 +55,12 @@ impl DatabaseDownloadsMetricRepository { /// # Errors /// /// Returns an [`Error`] if the database operation fails. - pub(crate) fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { - let torrent = self.load_torrent_downloads(info_hash)?; + pub(crate) async fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { + let torrent = self.load_torrent_downloads(info_hash).await?; match torrent { - Some(_number_of_downloads) => self.database.increase_downloads_for_torrent(info_hash), - None => self.save_torrent_downloads(info_hash, 1), + Some(_number_of_downloads) => self.database.increase_downloads_for_torrent(info_hash).await, + None => self.save_torrent_downloads(info_hash, 1).await, } } @@ -77,8 +72,8 @@ impl DatabaseDownloadsMetricRepository { /// # Errors /// /// Returns an [`Error`] if the underlying database query fails. - pub(crate) fn load_all_torrents_downloads(&self) -> Result { - self.database.load_all_torrents_downloads() + pub(crate) async fn load_all_torrents_downloads(&self) -> Result { + self.database.load_all_torrents_downloads().await } /// Loads one persistent torrent metrics from the database. @@ -89,8 +84,8 @@ impl DatabaseDownloadsMetricRepository { /// # Errors /// /// Returns an [`Error`] if the underlying database query fails. - pub(crate) fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { - self.database.load_torrent_downloads(info_hash) + pub(crate) async fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { + self.database.load_torrent_downloads(info_hash).await } /// Saves the persistent torrent metric into the database. @@ -106,8 +101,8 @@ impl DatabaseDownloadsMetricRepository { /// # Errors /// /// Returns an [`Error`] if the database operation fails. - pub(crate) fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: u32) -> Result<(), Error> { - self.database.save_torrent_downloads(info_hash, downloaded) + pub(crate) async fn save_torrent_downloads(&self, info_hash: &InfoHash, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.database.save_torrent_downloads(info_hash, downloaded).await } // Aggregate Metrics @@ -119,12 +114,12 @@ impl DatabaseDownloadsMetricRepository { /// # Errors /// /// Returns an [`Error`] if the database operation fails. - pub(crate) fn increase_global_downloads(&self) -> Result<(), Error> { - let torrent = self.database.load_global_downloads()?; + pub(crate) async fn increase_global_downloads(&self) -> Result<(), Error> { + let torrent = self.database.load_global_downloads().await?; match torrent { - Some(_number_of_downloads) => self.database.increase_global_downloads(), - None => self.database.save_global_downloads(1), + Some(_number_of_downloads) => self.database.increase_global_downloads().await, + None => self.database.save_global_downloads(1).await, } } @@ -133,8 +128,8 @@ impl DatabaseDownloadsMetricRepository { /// # Errors /// /// Returns an [`Error`] if the underlying database query fails. - pub(crate) fn load_global_downloads(&self) -> Result, Error> { - self.database.load_global_downloads() + pub(crate) async fn load_global_downloads(&self) -> Result, Error> { + self.database.load_global_downloads().await } } @@ -150,50 +145,50 @@ mod tests { fn initialize_db_persistent_torrent_repository() -> DatabaseDownloadsMetricRepository { let config = ephemeral_configuration(); let database = initialize_database(&config); - DatabaseDownloadsMetricRepository::new(&database) + DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store()) } - #[test] - fn it_saves_the_numbers_of_downloads_for_a_torrent_into_the_database() { + #[tokio::test] + async fn it_saves_the_numbers_of_downloads_for_a_torrent_into_the_database() { let repository = initialize_db_persistent_torrent_repository(); let infohash = sample_info_hash(); - repository.save_torrent_downloads(&infohash, 1).unwrap(); + repository.save_torrent_downloads(&infohash, 1).await.unwrap(); - let torrents = repository.load_all_torrents_downloads().unwrap(); + let torrents = repository.load_all_torrents_downloads().await.unwrap(); - assert_eq!(torrents.get(&infohash), Some(1).as_ref()); + assert_eq!(torrents.get(&infohash), Some(1_u64).as_ref()); } - #[test] - fn it_increases_the_numbers_of_downloads_for_a_torrent_into_the_database() { + #[tokio::test] + async fn it_increases_the_numbers_of_downloads_for_a_torrent_into_the_database() { let repository = initialize_db_persistent_torrent_repository(); let infohash = sample_info_hash(); - repository.increase_downloads_for_torrent(&infohash).unwrap(); + repository.increase_downloads_for_torrent(&infohash).await.unwrap(); - let torrents = repository.load_all_torrents_downloads().unwrap(); + let torrents = repository.load_all_torrents_downloads().await.unwrap(); - assert_eq!(torrents.get(&infohash), Some(1).as_ref()); + assert_eq!(torrents.get(&infohash), Some(1_u64).as_ref()); } - #[test] - fn it_loads_the_numbers_of_downloads_for_all_torrents_from_the_database() { + #[tokio::test] + async fn it_loads_the_numbers_of_downloads_for_all_torrents_from_the_database() { let repository = initialize_db_persistent_torrent_repository(); let infohash_one = sample_info_hash_one(); let infohash_two = sample_info_hash_two(); - repository.save_torrent_downloads(&infohash_one, 1).unwrap(); - repository.save_torrent_downloads(&infohash_two, 2).unwrap(); + repository.save_torrent_downloads(&infohash_one, 1).await.unwrap(); + repository.save_torrent_downloads(&infohash_two, 2).await.unwrap(); - let torrents = repository.load_all_torrents_downloads().unwrap(); + let torrents = repository.load_all_torrents_downloads().await.unwrap(); let mut expected_torrents = NumberOfDownloadsBTreeMap::new(); - expected_torrents.insert(infohash_one, 1); - expected_torrents.insert(infohash_two, 2); + expected_torrents.insert(infohash_one, 1_u64); + expected_torrents.insert(infohash_two, 2_u64); assert_eq!(torrents, expected_torrents); } diff --git a/packages/tracker-core/src/statistics/persisted/mod.rs b/packages/tracker-core/src/statistics/persisted/mod.rs index 86c28370d..6337cca75 100644 --- a/packages/tracker-core/src/statistics/persisted/mod.rs +++ b/packages/tracker-core/src/statistics/persisted/mod.rs @@ -23,12 +23,12 @@ pub async fn load_persisted_metrics( db_downloads_metric_repository: &Arc, now: DurationSinceUnixEpoch, ) -> Result<(), Error> { - if let Some(downloads) = db_downloads_metric_repository.load_global_downloads()? { + if let Some(downloads) = db_downloads_metric_repository.load_global_downloads().await? { stats_repository .set_counter( &metric_name!(TRACKER_CORE_PERSISTENT_TORRENTS_DOWNLOADS_TOTAL), &LabelSet::default(), - u64::from(downloads), + downloads, now, ) .await?; diff --git a/packages/tracker-core/src/test_helpers.rs b/packages/tracker-core/src/test_helpers.rs index bf21e6f94..67267be74 100644 --- a/packages/tracker-core/src/test_helpers.rs +++ b/packages/tracker-core/src/test_helpers.rs @@ -137,7 +137,7 @@ pub(crate) mod tests { &in_memory_whitelist.clone(), )); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let announce_handler = Arc::new(AnnounceHandler::new( &config.core, diff --git a/packages/tracker-core/src/torrent/manager.rs b/packages/tracker-core/src/torrent/manager.rs index 5acc27980..47aa2162f 100644 --- a/packages/tracker-core/src/torrent/manager.rs +++ b/packages/tracker-core/src/torrent/manager.rs @@ -70,8 +70,8 @@ impl TorrentsManager { /// /// Returns a `databases::error::Error` if unable to load the persistent /// torrent data. - pub fn load_torrents_from_database(&self) -> Result<(), databases::error::Error> { - let persistent_torrents = self.db_downloads_metric_repository.load_all_torrents_downloads()?; + pub async fn load_torrents_from_database(&self) -> Result<(), databases::error::Error> { + let persistent_torrents = self.db_downloads_metric_repository.load_all_torrents_downloads().await?; self.in_memory_torrent_repository.import_persistent(&persistent_torrents); @@ -170,7 +170,8 @@ mod tests { let swarms = Arc::new(Registry::default()); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::new(swarms)); let database = initialize_database(&config); - let database_persistent_torrent_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let database_persistent_torrent_repository = + Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let torrents_manager = Arc::new(TorrentsManager::new( &config, @@ -197,9 +198,10 @@ mod tests { services .database_persistent_torrent_repository .save_torrent_downloads(&infohash, 1) + .await .unwrap(); - torrents_manager.load_torrents_from_database().unwrap(); + torrents_manager.load_torrents_from_database().await.unwrap(); assert_eq!( services diff --git a/packages/tracker-core/src/whitelist/manager.rs b/packages/tracker-core/src/whitelist/manager.rs index 452fcb6c5..cc0f1458e 100644 --- a/packages/tracker-core/src/whitelist/manager.rs +++ b/packages/tracker-core/src/whitelist/manager.rs @@ -50,7 +50,7 @@ impl WhitelistManager { /// # Errors /// Returns a `database::Error` if the operation fails in the database. pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - self.database_whitelist.add(info_hash)?; + self.database_whitelist.add(info_hash).await?; self.in_memory_whitelist.add(info_hash).await; Ok(()) } @@ -63,7 +63,7 @@ impl WhitelistManager { /// # Errors /// Returns a `database::Error` if the operation fails in the database. pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - self.database_whitelist.remove(info_hash)?; + self.database_whitelist.remove(info_hash).await?; self.in_memory_whitelist.remove(info_hash).await; Ok(()) } @@ -76,7 +76,7 @@ impl WhitelistManager { /// # Errors /// Returns a `database::Error` if the operation fails to load from the database. pub async fn load_whitelist_from_database(&self) -> Result<(), databases::error::Error> { - let whitelisted_torrents_from_database = self.database_whitelist.load_from_database()?; + let whitelisted_torrents_from_database = self.database_whitelist.load_from_database().await?; self.in_memory_whitelist.clear().await; @@ -96,14 +96,13 @@ mod tests { use torrust_tracker_configuration::Core; use crate::databases::setup::initialize_database; - use crate::databases::Database; use crate::test_helpers::tests::ephemeral_configuration_for_listed_tracker; use crate::whitelist::manager::WhitelistManager; use crate::whitelist::repository::in_memory::InMemoryWhitelist; use crate::whitelist::repository::persisted::DatabaseWhitelist; struct WhitelistManagerDeps { - pub _database: Arc>, + pub _database: crate::databases::Persistence, pub database_whitelist: Arc, pub in_memory_whitelist: Arc, } @@ -115,7 +114,7 @@ mod tests { fn initialize_whitelist_manager_and_deps(config: &Core) -> (Arc, Arc) { let database = initialize_database(config); - let database_whitelist = Arc::new(DatabaseWhitelist::new(database.clone())); + let database_whitelist = Arc::new(DatabaseWhitelist::new(database.whitelist_store())); let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_manager = Arc::new(WhitelistManager::new(database_whitelist.clone(), in_memory_whitelist.clone())); @@ -145,7 +144,12 @@ mod tests { whitelist_manager.add_torrent_to_whitelist(&info_hash).await.unwrap(); assert!(services.in_memory_whitelist.contains(&info_hash).await); - assert!(services.database_whitelist.load_from_database().unwrap().contains(&info_hash)); + assert!(services + .database_whitelist + .load_from_database() + .await + .unwrap() + .contains(&info_hash)); } #[tokio::test] @@ -159,7 +163,12 @@ mod tests { whitelist_manager.remove_torrent_from_whitelist(&info_hash).await.unwrap(); assert!(!services.in_memory_whitelist.contains(&info_hash).await); - assert!(!services.database_whitelist.load_from_database().unwrap().contains(&info_hash)); + assert!(!services + .database_whitelist + .load_from_database() + .await + .unwrap() + .contains(&info_hash)); } mod persistence { @@ -172,7 +181,7 @@ mod tests { let info_hash = sample_info_hash(); - services.database_whitelist.add(&info_hash).unwrap(); + services.database_whitelist.add(&info_hash).await.unwrap(); whitelist_manager.load_whitelist_from_database().await.unwrap(); diff --git a/packages/tracker-core/src/whitelist/repository/persisted.rs b/packages/tracker-core/src/whitelist/repository/persisted.rs index eec6704d6..751214d02 100644 --- a/packages/tracker-core/src/whitelist/repository/persisted.rs +++ b/packages/tracker-core/src/whitelist/repository/persisted.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bittorrent_primitives::info_hash::InfoHash; -use crate::databases::{self, Database}; +use crate::databases::{self, WhitelistStore}; /// The persisted list of allowed torrents. /// @@ -12,13 +12,13 @@ use crate::databases::{self, Database}; pub struct DatabaseWhitelist { /// A database driver implementation: [`Sqlite3`](crate::core::databases::sqlite) /// or [`MySQL`](crate::core::databases::mysql) - database: Arc>, + database: Arc, } impl DatabaseWhitelist { /// Creates a new `DatabaseWhitelist`. #[must_use] - pub fn new(database: Arc>) -> Self { + pub fn new(database: Arc) -> Self { Self { database } } @@ -27,14 +27,14 @@ impl DatabaseWhitelist { /// # Errors /// Returns a `database::Error` if unable to add the `info_hash` to the /// whitelist. - pub(crate) fn add(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - let is_whitelisted = self.database.is_info_hash_whitelisted(*info_hash)?; + pub(crate) async fn add(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { + let is_whitelisted = self.database.is_info_hash_whitelisted(*info_hash).await?; if is_whitelisted { return Ok(()); } - self.database.add_info_hash_to_whitelist(*info_hash)?; + self.database.add_info_hash_to_whitelist(*info_hash).await?; Ok(()) } @@ -43,14 +43,14 @@ impl DatabaseWhitelist { /// /// # Errors /// Returns a `database::Error` if unable to remove the `info_hash`. - pub(crate) fn remove(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - let is_whitelisted = self.database.is_info_hash_whitelisted(*info_hash)?; + pub(crate) async fn remove(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { + let is_whitelisted = self.database.is_info_hash_whitelisted(*info_hash).await?; if !is_whitelisted { return Ok(()); } - self.database.remove_info_hash_from_whitelist(*info_hash)?; + self.database.remove_info_hash_from_whitelist(*info_hash).await?; Ok(()) } @@ -60,8 +60,8 @@ impl DatabaseWhitelist { /// # Errors /// Returns a `database::Error` if unable to load whitelisted `info_hash` /// values. - pub(crate) fn load_from_database(&self) -> Result, databases::error::Error> { - self.database.load_whitelist() + pub(crate) async fn load_from_database(&self) -> Result, databases::error::Error> { + self.database.load_whitelist().await } } @@ -76,65 +76,65 @@ mod tests { fn initialize_database_whitelist() -> DatabaseWhitelist { let configuration = ephemeral_configuration_for_listed_tracker(); let database = initialize_database(&configuration); - DatabaseWhitelist::new(database) + DatabaseWhitelist::new(database.whitelist_store()) } - #[test] - fn should_add_a_new_infohash_to_the_list() { + #[tokio::test] + async fn should_add_a_new_infohash_to_the_list() { let whitelist = initialize_database_whitelist(); let infohash = sample_info_hash(); - let _result = whitelist.add(&infohash); + let _result = whitelist.add(&infohash).await; - assert_eq!(whitelist.load_from_database().unwrap(), vec!(infohash)); + assert_eq!(whitelist.load_from_database().await.unwrap(), vec!(infohash)); } - #[test] - fn should_remove_a_infohash_from_the_list() { + #[tokio::test] + async fn should_remove_a_infohash_from_the_list() { let whitelist = initialize_database_whitelist(); let infohash = sample_info_hash(); - let _result = whitelist.add(&infohash); + let _result = whitelist.add(&infohash).await; - let _result = whitelist.remove(&infohash); + let _result = whitelist.remove(&infohash).await; - assert_eq!(whitelist.load_from_database().unwrap(), vec!()); + assert_eq!(whitelist.load_from_database().await.unwrap(), vec!()); } - #[test] - fn should_load_all_infohashes_from_the_database() { + #[tokio::test] + async fn should_load_all_infohashes_from_the_database() { let whitelist = initialize_database_whitelist(); let infohash = sample_info_hash(); - let _result = whitelist.add(&infohash); + let _result = whitelist.add(&infohash).await; - let result = whitelist.load_from_database().unwrap(); + let result = whitelist.load_from_database().await.unwrap(); assert_eq!(result, vec!(infohash)); } - #[test] - fn should_not_add_the_same_infohash_to_the_list_twice() { + #[tokio::test] + async fn should_not_add_the_same_infohash_to_the_list_twice() { let whitelist = initialize_database_whitelist(); let infohash = sample_info_hash(); - let _result = whitelist.add(&infohash); - let _result = whitelist.add(&infohash); + let _result = whitelist.add(&infohash).await; + let _result = whitelist.add(&infohash).await; - assert_eq!(whitelist.load_from_database().unwrap(), vec!(infohash)); + assert_eq!(whitelist.load_from_database().await.unwrap(), vec!(infohash)); } - #[test] - fn should_not_fail_removing_an_infohash_that_is_not_in_the_list() { + #[tokio::test] + async fn should_not_fail_removing_an_infohash_that_is_not_in_the_list() { let whitelist = initialize_database_whitelist(); let infohash = sample_info_hash(); - let result = whitelist.remove(&infohash); + let result = whitelist.remove(&infohash).await; assert!(result.is_ok()); } diff --git a/packages/tracker-core/src/whitelist/setup.rs b/packages/tracker-core/src/whitelist/setup.rs index cb18c1478..3926ab4cb 100644 --- a/packages/tracker-core/src/whitelist/setup.rs +++ b/packages/tracker-core/src/whitelist/setup.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use super::manager::WhitelistManager; use super::repository::in_memory::InMemoryWhitelist; use super::repository::persisted::DatabaseWhitelist; -use crate::databases::Database; +use crate::databases::WhitelistStore; /// Initializes the `WhitelistManager` by combining in-memory and database /// repositories. @@ -22,8 +22,8 @@ use crate::databases::Database; /// /// # Arguments /// -/// * `database` - An `Arc>` representing the database connection, -/// sed for persistent whitelist storage. +/// * `database` - A whitelist store handle used for persistent whitelist +/// storage. /// * `in_memory_whitelist` - An `Arc` representing the in-memory /// whitelist repository for fast access. /// @@ -33,7 +33,7 @@ use crate::databases::Database; /// whitelist repositories. #[must_use] pub fn initialize_whitelist_manager( - database: Arc>, + database: Arc, in_memory_whitelist: Arc, ) -> Arc { let database_whitelist = Arc::new(DatabaseWhitelist::new(database)); diff --git a/packages/tracker-core/src/whitelist/test_helpers.rs b/packages/tracker-core/src/whitelist/test_helpers.rs index cf1699be4..31add7111 100644 --- a/packages/tracker-core/src/whitelist/test_helpers.rs +++ b/packages/tracker-core/src/whitelist/test_helpers.rs @@ -21,7 +21,7 @@ pub(crate) mod tests { let database = initialize_database(&config.core); let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_authorization = Arc::new(WhitelistAuthorization::new(&config.core, &in_memory_whitelist.clone())); - let whitelist_manager = initialize_whitelist_manager(database.clone(), in_memory_whitelist.clone()); + let whitelist_manager = initialize_whitelist_manager(database.whitelist_store(), in_memory_whitelist.clone()); (whitelist_authorization, whitelist_manager) } diff --git a/packages/tracker-core/tests/common/test_env.rs b/packages/tracker-core/tests/common/test_env.rs index 3fe0464fe..2e31d9601 100644 --- a/packages/tracker-core/tests/common/test_env.rs +++ b/packages/tracker-core/tests/common/test_env.rs @@ -1,5 +1,6 @@ use std::net::IpAddr; use std::sync::Arc; +use std::time::Duration; use aquatic_udp_protocol::AnnounceEvent; use bittorrent_primitives::info_hash::InfoHash; @@ -149,6 +150,15 @@ impl TestEnv { let announce_data = self.announce_peer_completed(peer, remote_client_ip, info_hash).await; assert_eq!(announce_data.stats.downloads(), 1); + + if self + .tracker_core_container + .core_config + .tracker_policy + .persistent_torrent_completed_stat + { + self.wait_for_persisted_downloads(info_hash, 1).await; + } } pub async fn get_swarm_metadata(&self, info_hash: &InfoHash) -> Option { @@ -177,4 +187,29 @@ impl TestEnv { .unwrap() .value() } + + async fn wait_for_persisted_downloads(&self, info_hash: &InfoHash, expected: u64) { + const MAX_ATTEMPTS: usize = 50; + const RETRY_DELAY: Duration = Duration::from_millis(10); + let mut last_observed = None; + + for _attempt in 0..MAX_ATTEMPTS { + let downloads = self + .tracker_core_container + .database + .torrent_metrics_store() + .load_torrent_downloads(info_hash) + .await + .unwrap(); + + if downloads == Some(expected) { + return; + } + + last_observed = downloads; + tokio::time::sleep(RETRY_DELAY).await; + } + + panic!("timed out waiting for persisted downloads for torrent: expected {expected}, last saw {last_observed:?}"); + } } diff --git a/packages/tracker-core/tests/integration.rs b/packages/tracker-core/tests/integration.rs index b170aaebd..70904f217 100644 --- a/packages/tracker-core/tests/integration.rs +++ b/packages/tracker-core/tests/integration.rs @@ -82,6 +82,7 @@ async fn it_should_persist_the_number_of_completed_peers_for_each_torrent_into_t .tracker_core_container .torrents_manager .load_torrents_from_database() + .await .unwrap(); assert!(test_env.get_swarm_metadata(&info_hash).await.unwrap().downloads() == 1); diff --git a/packages/udp-tracker-server/src/handlers/announce.rs b/packages/udp-tracker-server/src/handlers/announce.rs index ea19611ce..34f060372 100644 --- a/packages/udp-tracker-server/src/handlers/announce.rs +++ b/packages/udp-tracker-server/src/handlers/announce.rs @@ -896,7 +896,8 @@ pub(crate) mod tests { let whitelist_authorization = Arc::new(WhitelistAuthorization::new(&config.core, &in_memory_whitelist.clone())); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = + Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let request = AnnounceRequestBuilder::default() .with_connection_id(make(gen_remote_fingerprint(&client_socket_addr), sample_issue_time()).unwrap()) diff --git a/packages/udp-tracker-server/src/handlers/mod.rs b/packages/udp-tracker-server/src/handlers/mod.rs index add576a89..ba391dfd9 100644 --- a/packages/udp-tracker-server/src/handlers/mod.rs +++ b/packages/udp-tracker-server/src/handlers/mod.rs @@ -273,7 +273,7 @@ pub(crate) mod tests { let in_memory_whitelist = Arc::new(InMemoryWhitelist::default()); let whitelist_authorization = Arc::new(WhitelistAuthorization::new(&config.core, &in_memory_whitelist.clone())); let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); - let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(&database)); + let db_downloads_metric_repository = Arc::new(DatabaseDownloadsMetricRepository::new(database.torrent_metrics_store())); let announce_handler = Arc::new(AnnounceHandler::new( &config.core, &whitelist_authorization, diff --git a/packages/udp-tracker-server/src/handlers/scrape.rs b/packages/udp-tracker-server/src/handlers/scrape.rs index 8bac05c1e..28e1a8f0f 100644 --- a/packages/udp-tracker-server/src/handlers/scrape.rs +++ b/packages/udp-tracker-server/src/handlers/scrape.rs @@ -10,6 +10,7 @@ use bittorrent_udp_tracker_core::services::scrape::ScrapeService; use bittorrent_udp_tracker_core::{self}; use torrust_tracker_primitives::core::ScrapeData; use torrust_tracker_primitives::service_binding::ServiceBinding; +use torrust_tracker_primitives::NumberOfDownloads as PersistentDownloadCount; use tracing::{instrument, Level}; use zerocopy::network_endian::I32; @@ -59,12 +60,11 @@ fn build_response(request: &ScrapeRequest, scrape_data: &ScrapeData) -> Response for file in &scrape_data.files { let swarm_metadata = file.1; - #[allow(clippy::cast_possible_truncation)] let scrape_entry = { TorrentScrapeStatistics { - seeders: NumberOfPeers(I32::new(i64::from(swarm_metadata.complete) as i32)), - completed: NumberOfDownloads(I32::new(i64::from(swarm_metadata.downloaded) as i32)), - leechers: NumberOfPeers(I32::new(i64::from(swarm_metadata.incomplete) as i32)), + seeders: NumberOfPeers(I32::new(udp_counter_from_u32(swarm_metadata.complete))), + completed: NumberOfDownloads(I32::new(udp_counter_from_downloads(swarm_metadata.downloaded))), + leechers: NumberOfPeers(I32::new(udp_counter_from_u32(swarm_metadata.incomplete))), } }; @@ -79,6 +79,14 @@ fn build_response(request: &ScrapeRequest, scrape_data: &ScrapeData) -> Response Response::from(response) } +fn udp_counter_from_u32(value: u32) -> i32 { + i32::try_from(value).unwrap_or(i32::MAX) +} + +fn udp_counter_from_downloads(value: PersistentDownloadCount) -> i32 { + value.min(i32::MAX as u64) as i32 +} + #[cfg(test)] mod tests { @@ -457,5 +465,12 @@ mod tests { .unwrap(); } } + + #[test] + fn should_saturate_large_download_counts_for_udp_protocol() { + assert_eq!(super::super::udp_counter_from_downloads(u64::MAX), i32::MAX); + assert_eq!(super::super::udp_counter_from_downloads((i32::MAX as u64) + 1), i32::MAX); + assert_eq!(super::super::udp_counter_from_downloads(42), 42); + } } } diff --git a/share/container/entry_script_sh b/share/container/entry_script_sh index 32cdfe33d..eb4ebce14 100644 --- a/share/container/entry_script_sh +++ b/share/container/entry_script_sh @@ -42,9 +42,16 @@ if [ -n "$TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER" ]; then # Select default MySQL configuration default_config="/usr/share/torrust/default/config/tracker.container.mysql.toml" + elif cmp_lc "$TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER" "postgresql"; then + + # (no database file needed for PostgreSQL) + + # Select default PostgreSQL configuration + default_config="/usr/share/torrust/default/config/tracker.container.postgresql.toml" + else echo "Error: Unsupported Database Type: \"$TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER\"." - echo "Please Note: Supported Database Types: \"sqlite3\", \"mysql\"." + echo "Please Note: Supported Database Types: \"sqlite3\", \"mysql\", \"postgresql\"." exit 1 fi else diff --git a/share/default/config/tracker.container.postgresql.toml b/share/default/config/tracker.container.postgresql.toml new file mode 100644 index 000000000..d7436f567 --- /dev/null +++ b/share/default/config/tracker.container.postgresql.toml @@ -0,0 +1,31 @@ +[metadata] +app = "torrust-tracker" +purpose = "configuration" +schema_version = "2.0.0" + +[logging] +threshold = "info" + +[core] +listed = false +private = false + +[core.database] +driver = "postgresql" +# Example credentials for local development and container smoke tests only. +# Do not reuse them in production deployments. +path = "postgresql://postgres:postgres@postgres:5432/torrust_tracker" + +# Uncomment to enable services + +#[[udp_trackers]] +#bind_address = "0.0.0.0:6969" + +#[[http_trackers]] +#bind_address = "0.0.0.0:7070" + +#[http_api] +#bind_address = "0.0.0.0:1212" + +#[http_api.access_tokens] +#admin = "MyAccessToken"