From 8cf0aa135fdb26d8729da390a7d7846bf16c75d0 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Tue, 28 Apr 2026 12:53:39 +0100 Subject: [PATCH 1/5] chore(docs): rename issue spec to include GitHub issue number prefix --- ...e-benchmarking.md => 1710-1525-03-persistence-benchmarking.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/issues/{1525-03-persistence-benchmarking.md => 1710-1525-03-persistence-benchmarking.md} (100%) diff --git a/docs/issues/1525-03-persistence-benchmarking.md b/docs/issues/1710-1525-03-persistence-benchmarking.md similarity index 100% rename from docs/issues/1525-03-persistence-benchmarking.md rename to docs/issues/1710-1525-03-persistence-benchmarking.md From 16c9c8a4695d336a4531204913390a47b20d9468 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Tue, 28 Apr 2026 16:59:50 +0100 Subject: [PATCH 2/5] docs(1525-03): update persistence benchmarking spec to DB-driver-level approach Revise the spec to reflect the simplified DB-driver-level benchmarking approach: - Benchmark Database trait methods directly, not through HTTP API - Remove Docker Compose and image-swapping complexity - Place binary in packages/tracker-core (not workspace root) - Report count, best, median, worst per operation (no p95 or ops/sec) - Single --ops default (10) for fast local runs under 3 minutes - Run once per driver/version; git diff of committed reports is the before/after comparison - Document in docs/benchmarking.md --- .../1710-1525-03-persistence-benchmarking.md | 349 +++++++++--------- 1 file changed, 170 insertions(+), 179 deletions(-) diff --git a/docs/issues/1710-1525-03-persistence-benchmarking.md b/docs/issues/1710-1525-03-persistence-benchmarking.md index d1b3ec32b..12dcb202d 100644 --- a/docs/issues/1710-1525-03-persistence-benchmarking.md +++ b/docs/issues/1710-1525-03-persistence-benchmarking.md @@ -1,4 +1,4 @@ -# Subissue Draft for #1525-03: Add Persistence Benchmarking +# Issue #1710 / Subissue #1525-03: Add Persistence Benchmarking ## Goal @@ -12,221 +12,207 @@ already covered by tests, otherwise performance comparisons risk masking regress ## Scope -- Implement the benchmark runner in Rust (a new binary, consistent with the `e2e_tests_runner` - pattern), following the same docker compose approach used in subissue #1525-02. -- Use one docker compose file per database backend. Each compose file defines the database - container and the tracker container together. The runner launches the compose stack, - discovers the ports, runs the workloads, and tears down. No manual `docker run` calls. +- Implement the benchmark runner as a binary inside `packages/tracker-core`, the package + that owns the persistence layer. No Docker Compose, no image building or swapping. +- Benchmark every method of the `Database` trait directly, using real driver instances + (SQLite file on disk; MySQL container via testcontainers — the same mechanism already used + in the package's integration tests). - Run the benchmark against SQLite and MySQL only. PostgreSQL is not available yet; the runner must be designed so PostgreSQL can be added in subissue #1525-08 without redesign. -- The benchmark compares two tracker Docker images: a `bench-before` image and a `bench-after` - image. The tracker image tag is passed to compose via an environment variable so the runner - can swap it per variant. This allows the same compose files and runner to be re-used after - each subsequent subissue. -- On the first run (this subissue), before and after use the same image built from the current - `develop` HEAD, giving an identical-baseline comparison. The committed report records this. -- Commit the first benchmark report into `docs/benchmarks/` as a baseline reference. Re-run - and update the report in each subsequent subissue that changes persistence behavior. +- One invocation produces results for one driver/version combination. Run it three times to + cover `sqlite3`, `mysql:8.0`, and `mysql:8.4`. +- Commit one JSON report per combination under `docs/benchmarks/` as the baseline. Re-run + and update the reports in each subsequent subissue that changes persistence behavior. The + git diff of those JSON files is the before/after comparison. ## Measurement Tool Rationale -**Why not Criterion?** `criterion` is a micro-benchmark framework: it runs the same in-process -function thousands of times in a tight loop, applies warm-up phases, and performs statistical -outlier detection for nanosecond-to-millisecond measurements. It is the right tool for the -existing `torrent-repository-benchmarking` crate (in-memory data structures). It is the wrong -tool here because: +**Why not Criterion?** `criterion` is a micro-benchmark framework designed for in-process +function calls. It is the right tool for the existing `torrent-repository-benchmarking` crate +(in-memory data structures). It is the wrong tool here because: -- Each operation involves a real HTTP round-trip to a containerized tracker talking to a real - database. The overhead dwarfs what criterion's sampling model expects. -- We need _aggregate_ metrics across N concurrent workers (ops/sec, p95 latency), not per-call - statistics from a single thread. -- The before/after comparison is across two different Docker images, not across two functions - in the same process — criterion has no model for that. +- Each operation involves a real database round-trip via an `r2d2` connection pool. The + overhead and variance are orders of magnitude larger than what criterion's sampling model + expects. +- The before/after comparison spans different branches (and later, different driver + implementations), not two functions in the same process — criterion has no model for that. **What to use instead**: `std::time::Instant` per-call timing, collected into a `Vec`, -then sorted for percentile extraction. This is exactly what the Python reference script does. -For concurrency, spawn N OS threads via `std::thread::spawn` (one per worker up to -`--concurrency`), each running blocking `reqwest` calls in a loop. Join all threads and -collect their `Duration` measurements into a shared `Vec` for percentile computation. Do -not use `rayon` — its work-stealing pool is designed for CPU-bound tasks and will stall -under I/O-bound HTTP workloads. Output is written as JSON (via `serde_json`) and Markdown. - -## Reference Workflow - -The PR #1695 review branch includes a Python reference: - -- `contrib/dev-tools/qa/run-before-after-db-benchmark.py` - -That script defines the full benchmark approach: it starts a real tracker binary, starts -database containers with free ports, sends HTTP workloads concurrently, collects latency -percentiles and throughput, and prints a before/after comparison. The Rust implementation -must replicate this approach. - -### What the Python script measures - -- **Startup time** — how long the tracker takes to reach `200 OK` on the health endpoint, - measured for both an empty database and a populated database (after the workloads have run). -- **Workloads** (each run sequentially and concurrently): - - `announce_lifecycle` — HTTP `started` announce followed by `completed` announce for each - unique infohash - - `whitelist_add` — REST API `POST /api/v1/whitelist/{info_hash}` - - `whitelist_reload` — REST API `GET /api/v1/whitelist/reload` - - `auth_key_add` — REST API `POST /api/v1/keys` - - `auth_key_reload` — REST API `GET /api/v1/keys/reload` -- **Metrics per workload**: count, total time, ops/sec, mean latency, median latency, p95 - latency, min/max latency. -- **Comparison output**: startup speedup (after/before), ops/s speedup, p95 latency improvement - ratio for each workload × driver combination. +then sorted to extract `best`, `median`, and `worst`. No external stats crate is needed. +Output is JSON only (via `serde_json`). -## Proposed Branch +## What Gets Measured -- `1525-03-persistence-benchmarking` +Every method on the `Database` trait, grouped by category: -## Testing Principles +| Category | Methods | +| ----------------- | ------------------------------------------------------------------------------------------------------------------- | +| Torrent metrics | `save_torrent_downloads`, `load_torrent_downloads`, `load_all_torrents_downloads`, `increase_downloads_for_torrent` | +| Aggregate metrics | `save_global_downloads`, `load_global_downloads`, `increase_global_downloads` | +| Whitelist | `add_info_hash_to_whitelist`, `get_info_hash_from_whitelist`, `load_whitelist`, `remove_info_hash_from_whitelist` | +| Auth keys | `add_key_to_keys`, `get_key_from_keys`, `load_keys`, `remove_key_from_keys` | -- **Isolation**: Each run uses a unique compose project name (e.g. - `torrust-bench---`) so container names, networks, and volumes - never collide with a parallel invocation. This mirrors the isolation strategy in - subissue #1525-02. -- **Independent system resources**: Do not bind to fixed host ports. Discover the ports - assigned by compose using `docker compose port`. Place all temporary files (SQLite database - file, tracker config, logs) in a `tempfile`-managed directory that is removed on exit. -- **Cleanup**: Use a `RunningCompose` `Drop` guard (from the `DockerCompose` wrapper in - subissue #1525-02) to call `docker compose down --volumes` unconditionally on success, - failure, and panic. -- **Verified before done**: Run the benchmark in a clean environment and include the output in - the PR description alongside the committed report. +Each method is called `--ops N` times (default `10`). The collected `Vec` is sorted +to produce `count`, `best`, `median`, and `worst` per operation. -## Tasks +A default of `10` is deliberately small so a local run finishes well under 3 minutes. +Pass a larger `--ops` value when tighter statistics are needed. + +## What Is NOT Measured + +- **Startup time** — not a persistence-layer concern; constant across persistence refactors. +- **Concurrent throughput** — the existing drivers are synchronous (`r2d2`); a single-threaded + loop gives stable, comparable numbers. Concurrent load is relevant after the async `sqlx` + migration (subissue #1525-05), but even then the comparison should be single-threaded first. +- **HTTP roundtrip latency** — noise relative to what is being refactored. +- **Before/after image swapping** — the benchmark runs once per branch; the committed report + is the baseline; the git diff is the comparison. -### 1) Add docker compose files for each database backend +## Proposed Branch -Add one compose file per database under `contrib/dev-tools/bench/`: +- `1710-add-persistence-benchmarking` -- `compose.bench-sqlite3.yaml` — tracker service + a volume for the SQLite database file. -- `compose.bench-mysql.yaml` — tracker service + MySQL service. +## Testing Principles -Design notes: +- **Real drivers**: SQLite uses a temporary file on disk; MySQL uses a testcontainers + `GenericImage` — the same mechanism already present in the package's integration tests. +- **MySQL container lifecycle**: reuse the retry logic in + `packages/tracker-core/src/databases/driver/mod.rs` to wait for container readiness. +- **Cleanup**: the testcontainers container is dropped (and therefore stopped) automatically + when the `RunningMysqlContainer` goes out of scope. +- **Verified before done**: run the benchmark in a clean environment and include a copy of + the console output in the PR description alongside the committed JSON reports. -- Parameterize the tracker image tag with an env var (e.g. - `TORRUST_TRACKER_BENCH_IMAGE`, defaulting to `torrust-tracker:bench`) so the runner can - swap before/after images without editing the file. -- Set `TORRUST_TRACKER_CONFIG_TOML` via the compose `environment` key so the runner can inject - a generated config without mounting a file. -- Do not expose fixed host ports in the compose files; expose only the container ports and let - Docker assign ephemeral host ports. The runner discovers them with `docker compose port`. -- Ensure `healthcheck` is defined for each service so `docker compose up --wait` blocks until - everything is ready. +## Tasks -Acceptance criteria: +### 1) Implement the benchmark runner binary inside `packages/tracker-core` -- [ ] `docker compose -f compose.bench-sqlite3.yaml up --wait` starts successfully. -- [ ] `docker compose -f compose.bench-mysql.yaml up --wait` starts successfully. -- [ ] `docker compose -f down --volumes` leaves no orphaned resources. +Add a new binary and supporting module to the `bittorrent-tracker-core` package. -### 2) Implement the Rust benchmark runner binary +**New files:** -Add a new binary `src/bin/persistence_benchmark_runner.rs` following the `e2e_tests_runner` -pattern. Reuse the `DockerCompose` wrapper introduced in subissue #1525-02 at -`src/console/ci/compose.rs`. +```text +packages/tracker-core/src/bin/persistence_benchmark_runner.rs ← thin entry point (3 lines) +packages/tracker-core/src/bench/ + mod.rs ← module doc, re-exports + runner.rs ← CLI args (clap), orchestration, tracing init + driver_bench.rs ← driver setup, measurement loops, RawResults + metrics.rs ← Vec → OperationStats (count, best, median, worst) + report.rs ← OperationStats → JSON (serde_json) + types.rs ← newtype wrappers (BenchDriver, Ops, …) +``` -**Dependencies** — add to the workspace `Cargo.toml` and the binary's crate: +**Dependencies** — add only to `packages/tracker-core/Cargo.toml` (not the workspace root): ```toml -reqwest = { version = "...", features = ["blocking"] } -serde_json = { version = "..." } +clap = { version = "...", features = ["derive"] } +serde_json = { version = "..." } # already present; confirm it is not dev-only +anyhow = { version = "..." } +tracing = { version = "..." } # already present ``` -`rayon` is not needed (see the concurrent workloads approach below). Run `cargo machete` -after to verify no unused dependencies remain. - -**Architecture** — add a module `src/console/ci/bench/` containing: - -- `runner.rs` — main orchestration and CLI argument parsing -- `workloads.rs` — HTTP client calls for each workload (announce, whitelist, auth key) -- `metrics.rs` — `Instant`-based latency collection, sorting, percentile and throughput - computation (no external stats crate needed) -- `report.rs` — JSON (`serde_json`) and Markdown formatting - -**CLI arguments** (mirroring the Python script): - -- `--before-image ` — tracker Docker image for the "before" variant - (default: `torrust-tracker:bench`) -- `--after-image ` — tracker Docker image for the "after" variant - (default: same as `--before-image`) -- `--dbs ` — space/comma-separated list of drivers (default: `sqlite3 mysql`) -- `--mysql-version ` — MySQL Docker image tag (default `8.4`) -- `--ops ` — number of operations per workload (default `200`) -- `--reload-iterations ` — iterations for reload workloads (default `30`) -- `--concurrency ` — worker threads for concurrent workloads (default `16`) -- `--json-output ` — write machine-readable JSON to this path -- `--report-output ` — write the human-readable Markdown report to this path - -**Per-suite lifecycle** (one suite = one `(driver, variant)` pair): - -1. Select the compose file for the driver. -2. Build or tag the tracker image as `TORRUST_TRACKER_BENCH_IMAGE` for this variant. -3. Create a unique compose project name. -4. `DockerCompose::up()` — blocks until all services are healthy. -5. Discover the tracker HTTP, REST API, and health check host ports via - `DockerCompose::port()`. -6. Record `startup_empty_ms` (time from `up` call to first successful health check response). -7. Run a warm-up iteration. -8. Run each workload sequentially then concurrently; collect per-operation `Duration` values. -9. Restart the tracker service only (or call `down` then `up` again) to measure - `startup_populated_ms` against the now-populated database. -10. `DockerCompose::down()` — unconditional, via `Drop` guard. - -**HTTP client**: use `reqwest` (blocking feature) for workload calls. - -**Concurrent workloads**: spawn `--concurrency` OS threads via `std::thread::spawn`, each -running blocking `reqwest` calls in a loop; collect per-thread `Duration` measurements into -a shared `Vec` (via `Arc>>` or join handles). Do not use `rayon` — -its work-stealing pool blocks under I/O-bound workloads. +Run `cargo machete` after to verify no unused dependencies remain. + +**CLI:** + +```text +cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ + --driver sqlite3|mysql # exactly one driver per run + --db-version 8.4 # DB image tag; ignored for sqlite3; default "8.4" for mysql + --ops 10 # samples per operation; default 10 + --json-output # default: bench-results.json +``` + +**Driver setup:** + +- `sqlite3` — create a temporary file path; build the `r2d2_sqlite` pool; create tables. +- `mysql` — start a testcontainers `GenericImage` with the requested `--db-version` tag; + reuse the container readiness retry logic from + `packages/tracker-core/src/databases/driver/mod.rs`. + +**Measurement loop** (per operation): + +1. Prepare realistic input data (a random `InfoHash`, `AuthKey`, etc.). +2. Time each call with `std::time::Instant`. +3. Repeat `--ops` times; collect into a `Vec`. +4. Sort and derive `count`, `best`, `median`, `worst`. + +**JSON output schema:** + +```json +{ + "meta": { + "git_revision": "", + "driver": "sqlite3", + "db_version": "-", + "ops": 10, + "timestamp": "2026-04-28T12:00:00Z" + }, + "operations": [ + { + "name": "add_info_hash_to_whitelist", + "count": 10, + "best_us": 42, + "median_us": 55, + "worst_us": 120 + } + ] +} +``` Acceptance criteria: -- [ ] The binary runs successfully against SQLite and MySQL. -- [ ] Startup times (empty and populated) are recorded for each driver. -- [ ] All five workload families are measured sequentially and concurrently. -- [ ] JSON output schema matches the Python reference (`results`, `comparisons` keys). -- [ ] Human-readable Markdown report is produced. -- [ ] All compose stacks are cleaned up unconditionally via `Drop` guards. -- [ ] No hard-coded host ports; all ports are discovered via `docker compose port`. +- [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver sqlite3` + runs to completion and writes a JSON report. +- [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4` + runs to completion and writes a JSON report. +- [ ] JSON schema matches the structure above. +- [ ] `cargo machete` reports no unused dependencies. -### 3) Commit the baseline benchmark report +### 2) Commit the baseline benchmark reports -After the binary is working: +Run the binary once per driver/version combination on the current branch HEAD and commit the +resulting JSON files. Each subsequent subissue reruns the same commands and commits updated +reports alongside the code change. The git diff is the before/after comparison. -- Build a Docker image from the current `develop` HEAD: - `docker build -t torrust-tracker:bench .` -- Run the benchmark with `--before-image torrust-tracker:bench` and - `--after-image torrust-tracker:bench` (both pointing to the same freshly built image, - producing an identical-baseline comparison). -- Save the JSON output to `docs/benchmarks/baseline.json`. -- Save the Markdown report to `docs/benchmarks/baseline.md`. -- Commit both files as part of this subissue's PR. +```bash +cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ + --driver sqlite3 \ + --json-output docs/benchmarks/baseline-sqlite3.json -Acceptance criteria: +cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ + --driver mysql --db-version 8.0 \ + --json-output docs/benchmarks/baseline-mysql-8.0.json + +cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ + --driver mysql --db-version 8.4 \ + --json-output docs/benchmarks/baseline-mysql-8.4.json +``` -- [ ] `docs/benchmarks/baseline.json` and `docs/benchmarks/baseline.md` are committed. -- [ ] The Markdown report is readable without tooling and identifies the git revision used. +Acceptance criteria: -### 4) Document the workflow +- [ ] `docs/benchmarks/baseline-sqlite3.json`, `docs/benchmarks/baseline-mysql-8.0.json`, + and `docs/benchmarks/baseline-mysql-8.4.json` are committed. +- [ ] Each file identifies the git revision, driver, db-version, ops count, and timestamp. -Steps: +### 3) Document the workflow -- Document how to invoke the benchmark locally. -- Document how to produce an updated report after each subsequent subissue. -- Note that PostgreSQL support will be added to the benchmark in subissue #1525-08. +- Add a section to `docs/benchmarking.md` explaining how to invoke the benchmark locally, how + to interpret the JSON output, and how to produce an updated report after each subsequent + subissue. +- Note that PostgreSQL support will be added in subissue #1525-08. Acceptance criteria: -- [ ] The benchmark is documented and runnable without ad hoc manual steps. +- [ ] `docs/benchmarking.md` documents the full workflow without ad hoc manual steps. ## Out of Scope - PostgreSQL support (reserved for subissue #1525-08). +- Concurrent throughput measurement (deferred until after the async `sqlx` migration in + subissue #1525-05). +- Startup time measurement (not a persistence-layer concern). +- HTTP-level benchmarking (noise relative to what is being refactored). - Defining hard performance gates for CI. - Replacing correctness-focused tests. - The existing `torrent-repository-benchmarking` criterion micro-benchmarks (those measure @@ -234,18 +220,23 @@ Acceptance criteria: ## Definition of Done +- [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver sqlite3` + runs to completion and prints a summary. +- [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4` + runs to completion and prints a summary. +- [ ] `docs/benchmarks/baseline-sqlite3.json`, `docs/benchmarks/baseline-mysql-8.0.json`, + and `docs/benchmarks/baseline-mysql-8.4.json` are committed. +- [ ] `docs/benchmarking.md` documents the workflow. - [ ] `cargo test --workspace --all-targets` passes. - [ ] `linter all` exits with code `0`. -- [ ] The benchmark has been executed successfully; `docs/benchmarks/baseline.md` and - `docs/benchmarks/baseline.json` are committed. - [ ] A passing run log is included in the PR description. ## References - EPIC: #1525 -- Reference PR: #1695 -- Reference implementation branch: `josecelano:pr-1684-review` — see EPIC for checkout - instructions (`docs/issues/1525-overhaul-persistence.md`) -- Reference script: `contrib/dev-tools/qa/run-before-after-db-benchmark.py` -- Docker compose wrapper: `src/console/ci/e2e/docker.rs` (pattern reused for compose wrapper) -- Subissue #1525-02 compose wrapper: `src/console/ci/compose.rs` +- GitHub issue: #1710 +- Existing driver test infrastructure: `packages/tracker-core/src/databases/driver/mod.rs` +- MySQL container helper: `packages/tracker-core/src/databases/driver/mysql.rs` + (`StoppedMysqlContainer`, `RunningMysqlContainer`) +- Style reference for binary layout: `src/console/ci/qbittorrent_e2e/runner.rs` +- Benchmarking docs: `docs/benchmarking.md` From 51c27fda813876afc1cb26ea1d5bbb0fa49dfdd2 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Tue, 28 Apr 2026 19:30:40 +0100 Subject: [PATCH 3/5] feat(tracker-core): add persistence benchmark runner Add the persistence_benchmark_runner binary for tracker-core with CLI options for driver, db version, and operation count. Implement benchmark orchestration and per-operation timing for torrent, whitelist, and auth key database operations across sqlite3 and mysql backends. Add JSON report generation with timing statistics and metadata, plus utility modules for metrics, types, sampling, and git revision capture. Update tracker-core dependencies/binary target, extend database driver parsing helpers, and document issue 1710 benchmarking implementation details. Closes #1710, part of #1525 --- .gitignore | 1 + Cargo.lock | 2 + .../1710-1525-03-persistence-benchmarking.md | 21 ++- packages/tracker-core/Cargo.toml | 4 +- .../driver_bench/database/mod.rs | 82 +++++++++ .../driver_bench/database/mysql.rs | 39 ++++ .../driver_bench/database/sqlite.rs | 22 +++ .../persistence_benchmark/driver_bench/mod.rs | 36 ++++ .../driver_bench/operations/keys.rs | 55 ++++++ .../driver_bench/operations/mod.rs | 32 ++++ .../driver_bench/operations/torrent.rs | 82 +++++++++ .../driver_bench/operations/whitelist.rs | 54 ++++++ .../driver_bench/sampling.rs | 50 ++++++ .../src/bin/persistence_benchmark/helpers.rs | 12 ++ .../src/bin/persistence_benchmark/metrics.rs | 101 +++++++++++ .../src/bin/persistence_benchmark/mod.rs | 10 ++ .../bin/persistence_benchmark/operations.rs | 20 +++ .../src/bin/persistence_benchmark/report.rs | 166 ++++++++++++++++++ .../bin/persistence_benchmark/reporting.rs | 84 +++++++++ .../src/bin/persistence_benchmark/runner.rs | 71 ++++++++ .../src/bin/persistence_benchmark/types.rs | 114 ++++++++++++ .../src/bin/persistence_benchmark_runner.rs | 76 ++++++++ .../tracker-core/src/databases/driver/mod.rs | 25 +++ 23 files changed, 1155 insertions(+), 4 deletions(-) create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/mod.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/helpers.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/metrics.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/mod.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/operations.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/report.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/reporting.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/runner.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/types.rs create mode 100644 packages/tracker-core/src/bin/persistence_benchmark_runner.rs diff --git a/.gitignore b/.gitignore index 4b811d59f..e6d0a9bfc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.code-workspace **/*.rs.bk /.coverage/ +/.benchmarks/ /.idea/ /.vscode/launch.json /data.db diff --git a/Cargo.lock b/Cargo.lock index 8e8d1db3c..e4dc3041e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -712,9 +712,11 @@ dependencies = [ name = "bittorrent-tracker-core" version = "3.0.0-develop" dependencies = [ + "anyhow", "aquatic_udp_protocol", "bittorrent-primitives", "chrono", + "clap", "derive_more", "local-ip-address", "mockall", diff --git a/docs/issues/1710-1525-03-persistence-benchmarking.md b/docs/issues/1710-1525-03-persistence-benchmarking.md index 12dcb202d..690ef75cd 100644 --- a/docs/issues/1710-1525-03-persistence-benchmarking.md +++ b/docs/issues/1710-1525-03-persistence-benchmarking.md @@ -14,6 +14,9 @@ already covered by tests, otherwise performance comparisons risk masking regress - Implement the benchmark runner as a binary inside `packages/tracker-core`, the package that owns the persistence layer. No Docker Compose, no image building or swapping. +- Keep the benchmark helper modules private to the binary target instead of exposing them from + the `bittorrent-tracker-core` library API. This keeps development tooling out of the + production module surface while still allowing `cargo run` execution from the same package. - Benchmark every method of the `Database` trait directly, using real driver instances (SQLite file on disk; MySQL container via testcontainers — the same mechanism already used in the package's integration tests). @@ -87,13 +90,25 @@ Pass a larger `--ops` value when tighter statistics are needed. ### 1) Implement the benchmark runner binary inside `packages/tracker-core` -Add a new binary and supporting module to the `bittorrent-tracker-core` package. +Add a new binary and binary-private support module tree to the `bittorrent-tracker-core` +package. + +**Module placement rationale:** + +- Do **not** expose the benchmark implementation from `packages/tracker-core/src/lib.rs`. + Benchmark orchestration is a developer tool, not part of the production library API. +- Do **not** place this implementation under `packages/tracker-core/benches/`. In this + repository, `benches/` is used for Criterion-style `cargo bench` targets. This persistence + runner is different: it has a CLI, writes JSON files, selects database drivers and versions, + and is intended to be run manually with `cargo run`. +- Therefore, keep the executable in `src/bin/` and place its helper modules under a + binary-private directory next to it. **New files:** ```text packages/tracker-core/src/bin/persistence_benchmark_runner.rs ← thin entry point (3 lines) -packages/tracker-core/src/bench/ +packages/tracker-core/src/bin/persistence_benchmark/ mod.rs ← module doc, re-exports runner.rs ← CLI args (clap), orchestration, tracing init driver_bench.rs ← driver setup, measurement loops, RawResults @@ -120,7 +135,7 @@ cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ --driver sqlite3|mysql # exactly one driver per run --db-version 8.4 # DB image tag; ignored for sqlite3; default "8.4" for mysql --ops 10 # samples per operation; default 10 - --json-output # default: bench-results.json + --json-output # default: .benchmarks/bench-results-[-].json ``` **Driver setup:** diff --git a/packages/tracker-core/Cargo.toml b/packages/tracker-core/Cargo.toml index 59c47dda2..3913283ff 100644 --- a/packages/tracker-core/Cargo.toml +++ b/packages/tracker-core/Cargo.toml @@ -18,9 +18,11 @@ default = [ ] db-compatibility-tests = [ ] [dependencies] +anyhow = "1" aquatic_udp_protocol = "0" bittorrent-primitives = "0.1.0" chrono = { version = "0", default-features = false, features = [ "clock" ] } +clap = { version = "4", features = [ "derive" ] } derive_more = { version = "2", features = [ "as_ref", "constructor", "from" ] } mockall = "0" r2d2 = "0" @@ -39,12 +41,12 @@ torrust-tracker-located-error = { version = "3.0.0-develop", path = "../located- torrust-tracker-metrics = { version = "3.0.0-develop", path = "../metrics" } torrust-tracker-primitives = { version = "3.0.0-develop", path = "../primitives" } torrust-tracker-swarm-coordination-registry = { version = "3.0.0-develop", path = "../swarm-coordination-registry" } +testcontainers = "0" tracing = "0" [dev-dependencies] local-ip-address = "0" mockall = "0" -testcontainers = "0" torrust-rest-tracker-api-client = { version = "3.0.0-develop", path = "../rest-tracker-api-client" } torrust-tracker-test-helpers = { version = "3.0.0-develop", path = "../test-helpers" } url = "2.5.4" diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs new file mode 100644 index 000000000..70f8142d5 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs @@ -0,0 +1,82 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{anyhow, Context, Result}; +use bittorrent_tracker_core::databases::driver::Driver; +use bittorrent_tracker_core::databases::Database; +use testcontainers::{ContainerAsync, GenericImage}; + +mod mysql; +mod sqlite; + +pub(super) struct ActiveDatabase { + pub(super) database: Arc>, + resource: Option, +} + +enum BenchmarkResource { + Sqlite(PathBuf), + Mysql(Box>), +} + +impl ActiveDatabase { + /// Creates an initialized benchmark database for the selected driver. + /// + /// For `sqlite3`, this creates a unique temporary database file. + /// For `mysql`, this starts a temporary container and builds a connection + /// URL from mapped host/port details. + /// + /// # Errors + /// + /// Returns an error if the `MySQL` container cannot be started or queried for + /// connection details. + pub(super) async fn new(driver: Driver, db_version: &str) -> Result { + match driver { + Driver::Sqlite3 => Ok(sqlite::initialize()), + Driver::MySQL => mysql::initialize(db_version).await, + } + } +} + +impl Drop for ActiveDatabase { + fn drop(&mut self) { + match self.resource.take() { + Some(BenchmarkResource::Sqlite(path)) => { + let _removed_file_result = std::fs::remove_file(path); + } + Some(BenchmarkResource::Mysql(container)) => { + drop(container); + } + None => {} + } + } +} + +pub(super) async fn reset_database(database: &dyn Database) -> Result<()> { + create_database_tables_with_retry(database).await?; + database + .drop_database_tables() + .context("failed to drop benchmark database tables")?; + create_database_tables_with_retry(database).await +} + +/// Retries table creation until the database is ready. +/// +/// This primarily shields `MySQL` startup latency where the process may be up +/// before it is ready to accept migrations/queries. +/// +/// # Errors +/// +/// Returns an error if the database is still not ready after all retries. +async fn create_database_tables_with_retry(database: &dyn Database) -> Result<()> { + for _ in 0..5 { + if database.create_database_tables().is_ok() { + return Ok(()); + } + + tokio::time::sleep(Duration::from_secs(2)).await; + } + + Err(anyhow!("database is not ready after retries")) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs new file mode 100644 index 000000000..3caad237f --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs @@ -0,0 +1,39 @@ +use anyhow::{Context, Result}; +use bittorrent_tracker_core::databases::setup::initialize_database; +use testcontainers::core::IntoContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; +use torrust_tracker_configuration as configuration; + +use super::{ActiveDatabase, BenchmarkResource}; + +pub(super) async fn initialize(db_version: &str) -> Result { + let mysql_container = GenericImage::new("mysql", db_version) + .with_exposed_port(3306.tcp()) + .with_env_var("MYSQL_ROOT_PASSWORD", "test") + .with_env_var("MYSQL_DATABASE", "torrust_tracker_bench") + .with_env_var("MYSQL_ROOT_HOST", "%") + .start() + .await + .context("failed to start mysql test container")?; + + let host = mysql_container + .get_host() + .await + .context("failed to resolve mysql container host")?; + let port = mysql_container + .get_host_port_ipv4(3306) + .await + .context("failed to resolve mysql container host port")?; + + let mysql_database_url = format!("mysql://root:test@{host}:{port}/torrust_tracker_bench"); + let mut config = configuration::Core::default(); + config.database.driver = configuration::Driver::MySQL; + config.database.path = mysql_database_url; + let database = initialize_database(&config); + + Ok(ActiveDatabase { + database, + resource: Some(BenchmarkResource::Mysql(Box::new(mysql_container))), + }) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs new file mode 100644 index 000000000..f597cc32b --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs @@ -0,0 +1,22 @@ +use bittorrent_tracker_core::databases::setup::initialize_database; +use torrust_tracker_configuration as configuration; + +use super::{ActiveDatabase, BenchmarkResource}; + +pub(super) fn initialize() -> ActiveDatabase { + let sqlite_db_path = std::env::temp_dir().join(format!( + "torrust-tracker-core-benchmark-{}.sqlite3", + chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default() + )); + let sqlite_db_path_as_string = sqlite_db_path.to_string_lossy().to_string(); + let mut config = configuration::Core::default(); + config.database.driver = configuration::Driver::Sqlite3; + config.database.path = sqlite_db_path_as_string; + + let database = initialize_database(&config); + + ActiveDatabase { + database, + resource: Some(BenchmarkResource::Sqlite(sqlite_db_path)), + } +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs new file mode 100644 index 000000000..674eb3428 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs @@ -0,0 +1,36 @@ +use std::time::Duration; + +use anyhow::Result; +use bittorrent_tracker_core::databases::driver::Driver; + +use super::types::OpsCount; + +mod database; +mod operations; +mod sampling; + +#[derive(Debug)] +pub struct RawOperationSamples { + pub name: String, + pub samples: Vec, +} + +/// Runs all persistence operation benchmarks for one driver/version pair. +/// +/// # Errors +/// +/// Returns an error if database setup fails or any benchmarked database +/// operation fails. +pub async fn run(driver: Driver, db_version: &str, ops: OpsCount) -> Result> { + let active_database = database::ActiveDatabase::new(driver, db_version).await?; + database::reset_database(active_database.database.as_ref().as_ref()).await?; + + let ops = ops.get(); + + let mut operations_samples = Vec::new(); + operations::benchmark_torrent_operations(active_database.database.as_ref().as_ref(), ops, &mut operations_samples)?; + operations::benchmark_whitelist_operations(active_database.database.as_ref().as_ref(), ops, &mut operations_samples)?; + operations::benchmark_key_operations(active_database.database.as_ref().as_ref(), ops, &mut operations_samples)?; + + Ok(operations_samples) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs new file mode 100644 index 000000000..388147cc2 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs @@ -0,0 +1,55 @@ +use anyhow::{Context, Result}; +use bittorrent_tracker_core::authentication; +use bittorrent_tracker_core::databases::Database; + +use super::super::sampling::measure_operation; +use super::super::RawOperationSamples; + +/// Benchmarks authentication-key persistence operations. +/// +/// # Errors +/// +/// Returns an error if any setup or measured database operation fails. +pub(super) fn benchmark_key_operations( + database: &dyn Database, + ops: usize, + operations: &mut Vec, +) -> Result<()> { + operations.push(measure_operation("add_key_to_keys", ops, |_| { + let peer_key = authentication::key::generate_key(None); + let _added_rows = database.add_key_to_keys(&peer_key).context("add_key_to_keys failed")?; + Ok(()) + })?); + + let persisted_peer_key = authentication::key::generate_key(None); + let _added_rows = database + .add_key_to_keys(&persisted_peer_key) + .context("failed to seed get_key_from_keys")?; + let persisted_key = persisted_peer_key.key(); + operations.push(measure_operation("get_key_from_keys", ops, |_| { + let persisted_key_result = database + .get_key_from_keys(&persisted_key) + .context("get_key_from_keys failed")?; + drop(persisted_key_result); + Ok(()) + })?); + + operations.push(measure_operation("load_keys", ops, |_| { + let keys = database.load_keys().context("load_keys failed")?; + drop(keys); + Ok(()) + })?); + + operations.push(measure_operation("remove_key_from_keys", ops, |_| { + let peer_key = authentication::key::generate_key(None); + let _added_rows = database + .add_key_to_keys(&peer_key) + .context("failed to seed remove_key_from_keys")?; + let _removed_rows = database + .remove_key_from_keys(&peer_key.key()) + .context("remove_key_from_keys failed")?; + Ok(()) + })?); + + Ok(()) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/mod.rs new file mode 100644 index 000000000..69ec5bc42 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/mod.rs @@ -0,0 +1,32 @@ +mod keys; +mod torrent; +mod whitelist; + +use anyhow::Result; +use bittorrent_tracker_core::databases::Database; + +use super::RawOperationSamples; + +pub(super) fn benchmark_torrent_operations( + database: &dyn Database, + ops: usize, + operations: &mut Vec, +) -> Result<()> { + torrent::benchmark_torrent_operations(database, ops, operations) +} + +pub(super) fn benchmark_whitelist_operations( + database: &dyn Database, + ops: usize, + operations: &mut Vec, +) -> Result<()> { + whitelist::benchmark_whitelist_operations(database, ops, operations) +} + +pub(super) fn benchmark_key_operations( + database: &dyn Database, + ops: usize, + operations: &mut Vec, +) -> Result<()> { + keys::benchmark_key_operations(database, ops, operations) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs new file mode 100644 index 000000000..ca7fb28b2 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs @@ -0,0 +1,82 @@ +use anyhow::{Context, Result}; +use bittorrent_tracker_core::databases::Database; + +use super::super::sampling::{downloads_from_index, info_hash_from_index, measure_operation}; +use super::super::RawOperationSamples; + +/// Benchmarks torrent statistics persistence operations. +/// +/// This function seeds prerequisite records where needed so each measured +/// operation executes on realistic state. +/// +/// # Errors +/// +/// Returns an error if any setup or measured database operation fails. +pub(super) fn benchmark_torrent_operations( + database: &dyn Database, + ops: usize, + operations: &mut Vec, +) -> Result<()> { + operations.push(measure_operation("save_torrent_downloads", ops, |index| { + let info_hash = info_hash_from_index(index + 1)?; + let downloads = downloads_from_index(index)?; + database + .save_torrent_downloads(&info_hash, downloads) + .context("save_torrent_downloads failed") + })?); + + let load_torrent_info_hash = info_hash_from_index(10_000)?; + database + .save_torrent_downloads(&load_torrent_info_hash, 123) + .context("failed to seed load_torrent_downloads")?; + operations.push(measure_operation("load_torrent_downloads", ops, |_| { + let _downloads_result = database + .load_torrent_downloads(&load_torrent_info_hash) + .context("load_torrent_downloads failed")?; + Ok(()) + })?); + + operations.push(measure_operation("load_all_torrents_downloads", ops, |_| { + let all_downloads = database + .load_all_torrents_downloads() + .context("load_all_torrents_downloads failed")?; + drop(all_downloads); + Ok(()) + })?); + + let increasing_downloads_info_hash = info_hash_from_index(20_000)?; + database + .save_torrent_downloads(&increasing_downloads_info_hash, 0) + .context("failed to seed increase_downloads_for_torrent")?; + operations.push(measure_operation("increase_downloads_for_torrent", ops, |_| { + database + .increase_downloads_for_torrent(&increasing_downloads_info_hash) + .context("increase_downloads_for_torrent failed") + })?); + + operations.push(measure_operation("save_global_downloads", ops, |index| { + let downloads = downloads_from_index(index)?; + database + .save_global_downloads(downloads) + .context("save_global_downloads failed") + })?); + + database + .save_global_downloads(0) + .context("failed to seed load_global_downloads")?; + operations.push(measure_operation("load_global_downloads", ops, |_| { + let _downloads_result = database.load_global_downloads().context("load_global_downloads failed")?; + Ok(()) + })?); + + database + .save_global_downloads(0) + .context("failed to seed increase_global_downloads")?; + operations.push(measure_operation("increase_global_downloads", ops, |_| { + database + .increase_global_downloads() + .context("increase_global_downloads failed") + })?); + + Ok(()) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs new file mode 100644 index 000000000..2efb25cb9 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs @@ -0,0 +1,54 @@ +use anyhow::{Context, Result}; +use bittorrent_tracker_core::databases::Database; + +use super::super::sampling::{info_hash_from_index, measure_operation}; +use super::super::RawOperationSamples; + +/// Benchmarks whitelist-related persistence operations. +/// +/// # Errors +/// +/// Returns an error if any setup or measured database operation fails. +pub(super) fn benchmark_whitelist_operations( + database: &dyn Database, + ops: usize, + operations: &mut Vec, +) -> Result<()> { + operations.push(measure_operation("add_info_hash_to_whitelist", ops, |index| { + let info_hash = info_hash_from_index(30_000 + index)?; + let _added_rows = database + .add_info_hash_to_whitelist(info_hash) + .context("add_info_hash_to_whitelist failed")?; + Ok(()) + })?); + + let whitelisted_info_hash = info_hash_from_index(40_000)?; + let _added_rows = database + .add_info_hash_to_whitelist(whitelisted_info_hash) + .context("failed to seed get_info_hash_from_whitelist")?; + operations.push(measure_operation("get_info_hash_from_whitelist", ops, |_| { + let _info_hash_result = database + .get_info_hash_from_whitelist(whitelisted_info_hash) + .context("get_info_hash_from_whitelist failed")?; + Ok(()) + })?); + + operations.push(measure_operation("load_whitelist", ops, |_| { + let whitelist = database.load_whitelist().context("load_whitelist failed")?; + drop(whitelist); + Ok(()) + })?); + + operations.push(measure_operation("remove_info_hash_from_whitelist", ops, |index| { + let info_hash = info_hash_from_index(50_000 + index)?; + let _added_rows = database + .add_info_hash_to_whitelist(info_hash) + .context("failed to seed remove_info_hash_from_whitelist")?; + let _removed_rows = database + .remove_info_hash_from_whitelist(info_hash) + .context("remove_info_hash_from_whitelist failed")?; + Ok(()) + })?); + + Ok(()) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs new file mode 100644 index 000000000..798c7ff8e --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs @@ -0,0 +1,50 @@ +use std::str::FromStr; +use std::time::Instant; + +use anyhow::{anyhow, Context, Result}; +use bittorrent_primitives::info_hash::InfoHash; + +use super::RawOperationSamples; + +/// Measures one database operation `ops` times and records elapsed samples. +/// +/// The closure receives the iteration index so callers can generate distinct +/// fixture values when required. +/// +/// # Errors +/// +/// Returns an error if any operation invocation fails. +pub(super) fn measure_operation(name: impl Into, ops: usize, mut operation: F) -> Result +where + F: FnMut(usize) -> Result<()>, +{ + let name = name.into(); + let mut samples = Vec::with_capacity(ops); + + for index in 0..ops { + let start = Instant::now(); + operation(index)?; + samples.push(start.elapsed()); + } + + Ok(RawOperationSamples { name, samples }) +} + +/// Converts a loop index into a valid download-count value. +/// +/// # Errors +/// +/// Returns an error if the index does not fit in `u32`. +pub(super) fn downloads_from_index(index: usize) -> Result { + u32::try_from(index).context("failed to convert operation index to download count") +} + +/// Builds a deterministic 40-hex-char `InfoHash` from an index. +/// +/// # Errors +/// +/// Returns an error if the generated value cannot be parsed as an `InfoHash`. +pub(super) fn info_hash_from_index(index: usize) -> Result { + let hex = format!("{index:040x}"); + InfoHash::from_str(&hex).map_err(|error| anyhow!("failed to generate benchmark info hash: {error:?}")) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/helpers.rs b/packages/tracker-core/src/bin/persistence_benchmark/helpers.rs new file mode 100644 index 000000000..d6474e118 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/helpers.rs @@ -0,0 +1,12 @@ +use std::process::Command; + +#[must_use] +pub fn git_revision() -> String { + match Command::new("git").args(["rev-parse", "HEAD"]).output() { + Ok(output) if output.status.success() => { + let revision = String::from_utf8_lossy(&output.stdout); + revision.trim().to_string() + } + _ => "unknown".to_string(), + } +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/metrics.rs b/packages/tracker-core/src/bin/persistence_benchmark/metrics.rs new file mode 100644 index 000000000..89e2d1049 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/metrics.rs @@ -0,0 +1,101 @@ +use std::time::Duration; + +use anyhow::{anyhow, Result}; + +use super::driver_bench::RawOperationSamples; + +#[derive(Debug, Clone)] +pub struct OperationStats { + pub name: String, + pub count: usize, + pub best: Duration, + pub median: Duration, + pub worst: Duration, +} + +/// Computes benchmark statistics for each operation. +/// +/// # Errors +/// +/// Returns an error if an operation has no samples. +pub fn compute(raw_operations: Vec) -> Result> { + let mut operation_stats = Vec::with_capacity(raw_operations.len()); + + for raw_operation in raw_operations { + operation_stats.push(compute_operation(raw_operation)?); + } + + Ok(operation_stats) +} + +/// Computes summary statistics for one benchmark operation. +/// +/// Samples are sorted so `best`/`median`/`worst` are deterministic and +/// independent from insertion order. +/// +/// # Errors +/// +/// Returns an error when no samples were collected for the operation. +fn compute_operation(raw_operation: RawOperationSamples) -> Result { + if raw_operation.samples.is_empty() { + return Err(anyhow!("operation '{}' has no samples", raw_operation.name)); + } + + let mut sorted_samples = raw_operation.samples; + sorted_samples.sort_unstable(); + + let count = sorted_samples.len(); + let best = sorted_samples[0]; + let median = sorted_samples[count / 2]; + let worst = sorted_samples[count - 1]; + + Ok(OperationStats { + name: raw_operation.name, + count, + best, + median, + worst, + }) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::compute; + use crate::persistence_benchmark::driver_bench::RawOperationSamples; + + #[test] + fn it_should_compute_sorted_best_median_and_worst_for_each_operation() { + let raw_operations = vec![RawOperationSamples { + name: "save_torrent_downloads".to_string(), + samples: vec![ + Duration::from_micros(50), + Duration::from_micros(20), + Duration::from_micros(30), + Duration::from_micros(10), + ], + }]; + + let stats = compute(raw_operations).expect("metrics should compute"); + + assert_eq!(stats.len(), 1); + assert_eq!(stats[0].name, "save_torrent_downloads"); + assert_eq!(stats[0].count, 4); + assert_eq!(stats[0].best, Duration::from_micros(10)); + assert_eq!(stats[0].median, Duration::from_micros(30)); + assert_eq!(stats[0].worst, Duration::from_micros(50)); + } + + #[test] + fn it_should_fail_when_operation_has_no_samples() { + let raw_operations = vec![RawOperationSamples { + name: "load_keys".to_string(), + samples: Vec::new(), + }]; + + let error = compute(raw_operations).expect_err("empty samples should fail"); + + assert_eq!(error.to_string(), "operation 'load_keys' has no samples"); + } +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/mod.rs new file mode 100644 index 000000000..57f565021 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/mod.rs @@ -0,0 +1,10 @@ +//! Binary-private support code for the persistence benchmark runner. + +pub mod driver_bench; +pub mod helpers; +pub mod metrics; +pub mod operations; +pub mod report; +pub mod reporting; +pub mod runner; +pub mod types; diff --git a/packages/tracker-core/src/bin/persistence_benchmark/operations.rs b/packages/tracker-core/src/bin/persistence_benchmark/operations.rs new file mode 100644 index 000000000..c75861ad4 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/operations.rs @@ -0,0 +1,20 @@ +use anyhow::Result; +use bittorrent_tracker_core::databases::driver::Driver; + +use super::types::{DbVersion, OpsCount}; +use super::{driver_bench, metrics}; + +/// Collects benchmark operation samples and computes aggregate statistics. +/// +/// # Errors +/// +/// Returns an error if operation sampling or metrics computation fails. +pub async fn collect_operation_stats( + driver: &Driver, + db_version: &DbVersion, + ops: OpsCount, +) -> Result> { + let raw_operations = driver_bench::run(driver.clone(), db_version.as_str(), ops).await?; + + metrics::compute(raw_operations) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/report.rs b/packages/tracker-core/src/bin/persistence_benchmark/report.rs new file mode 100644 index 000000000..9ea74d431 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/report.rs @@ -0,0 +1,166 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use serde::Serialize; + +use super::helpers; +use super::metrics::OperationStats; + +#[derive(Debug, Serialize)] +pub struct BenchReport { + pub meta: ReportMeta, + pub operations: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ReportMeta { + pub git_revision: String, + pub driver: String, + pub db_version: String, + pub ops: usize, + pub timestamp: String, + pub timings_ms: ReportTimings, +} + +#[derive(Debug, Serialize)] +pub struct ReportTimings { + pub benchmark: u64, + pub report_build: u64, + pub total: u64, +} + +#[derive(Debug, Serialize)] +pub struct OperationReport { + pub name: String, + pub count: usize, + pub best_us: u64, + pub median_us: u64, + pub worst_us: u64, +} + +impl BenchReport { + /// Builds a serializable benchmark report from aggregated operation stats. + /// + /// Durations are converted to microseconds to keep report values compact, + /// language-agnostic, and easy to compare across runs. + #[must_use] + pub fn new(meta: ReportMeta, operation_stats: Vec) -> Self { + let operations = operation_stats + .into_iter() + .map(|operation_stat| OperationReport { + name: operation_stat.name.clone(), + count: operation_stat.count, + best_us: duration_to_micros(operation_stat.best), + median_us: duration_to_micros(operation_stat.median), + worst_us: duration_to_micros(operation_stat.worst), + }) + .collect(); + + Self { meta, operations } + } +} + +impl ReportMeta { + /// Captures report metadata for one benchmark execution. + /// + /// The timestamp is recorded in RFC 3339 format and the git revision is + /// resolved from the current repository state. + #[must_use] + pub fn from_run_context(driver: &str, db_version: &str, ops: usize, timings_ms: ReportTimings) -> Self { + let git_revision = helpers::git_revision(); + + Self { + git_revision, + driver: driver.to_string(), + db_version: db_version.to_string(), + ops, + timestamp: Utc::now().to_rfc3339(), + timings_ms, + } + } +} + +/// Serializes the benchmark report as pretty-printed JSON. +/// +/// # Errors +/// +/// Returns an error if serialization fails. +pub fn to_json_pretty(report: &BenchReport) -> Result { + serde_json::to_string_pretty(report).context("failed to serialize benchmark report") +} + +/// Converts a duration into microseconds for JSON serialization. +/// +/// Saturates to `u64::MAX` if conversion overflows. +fn duration_to_micros(duration: std::time::Duration) -> u64 { + u64::try_from(duration.as_micros()).unwrap_or(u64::MAX) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{to_json_pretty, BenchReport, ReportMeta, ReportTimings}; + use crate::persistence_benchmark::metrics::OperationStats; + + #[test] + fn it_should_convert_operation_durations_to_microseconds_in_report() { + let meta = ReportMeta { + git_revision: "test-revision".to_string(), + driver: "sqlite3".to_string(), + db_version: "-".to_string(), + ops: 2, + timestamp: "2026-01-01T00:00:00+00:00".to_string(), + timings_ms: ReportTimings { + benchmark: 10, + report_build: 1, + total: 11, + }, + }; + let operation_stats = vec![OperationStats { + name: "save_global_downloads".to_string(), + count: 2, + best: Duration::from_micros(7), + median: Duration::from_micros(11), + worst: Duration::from_micros(19), + }]; + + let report = BenchReport::new(meta, operation_stats); + + assert_eq!(report.operations.len(), 1); + assert_eq!(report.operations[0].name, "save_global_downloads"); + assert_eq!(report.operations[0].best_us, 7); + assert_eq!(report.operations[0].median_us, 11); + assert_eq!(report.operations[0].worst_us, 19); + } + + #[test] + fn it_should_serialize_report_as_valid_pretty_json() { + let meta = ReportMeta { + git_revision: "test-revision".to_string(), + driver: "sqlite3".to_string(), + db_version: "-".to_string(), + ops: 1, + timestamp: "2026-01-01T00:00:00+00:00".to_string(), + timings_ms: ReportTimings { + benchmark: 5, + report_build: 1, + total: 6, + }, + }; + let operation_stats = vec![OperationStats { + name: "load_whitelist".to_string(), + count: 1, + best: Duration::from_micros(3), + median: Duration::from_micros(3), + worst: Duration::from_micros(3), + }]; + let report = BenchReport::new(meta, operation_stats); + + let json = to_json_pretty(&report).expect("report should serialize"); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("json should parse"); + + assert_eq!(parsed["meta"]["driver"], "sqlite3"); + assert_eq!(parsed["meta"]["timings_ms"]["total"], 6); + assert_eq!(parsed["operations"][0]["name"], "load_whitelist"); + } +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs b/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs new file mode 100644 index 000000000..10ea7ddb1 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs @@ -0,0 +1,84 @@ +use bittorrent_tracker_core::databases::driver::Driver; + +use super::types::DbVersion; +use super::{metrics, report}; + +/// Builds the final JSON-serializable report from run context and metrics. +/// +/// For `sqlite3` runs, `db_version` is normalized to `-` because there is no +/// image tag associated with the local file-backed database. +#[must_use] +pub fn build_report( + driver: &Driver, + db_version: &DbVersion, + ops: usize, + timings_ms: report::ReportTimings, + operation_stats: Vec, +) -> report::BenchReport { + let normalized_db_version = match driver { + Driver::Sqlite3 => "-".to_string(), + Driver::MySQL => db_version.to_string(), + }; + + let meta = report::ReportMeta::from_run_context(driver.as_str(), &normalized_db_version, ops, timings_ms); + + report::BenchReport::new(meta, operation_stats) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use std::time::Duration; + + use bittorrent_tracker_core::databases::driver::Driver; + + use super::build_report; + use crate::persistence_benchmark::metrics::OperationStats; + use crate::persistence_benchmark::report::ReportTimings; + use crate::persistence_benchmark::types::DbVersion; + + #[test] + fn it_should_normalize_db_version_to_dash_for_sqlite_reports() { + let db_version = DbVersion::from_str("8.4").expect("db version should parse"); + let timings_ms = ReportTimings { + benchmark: 7, + report_build: 1, + total: 8, + }; + let operation_stats = vec![OperationStats { + name: "save_torrent_downloads".to_string(), + count: 1, + best: Duration::from_micros(1), + median: Duration::from_micros(1), + worst: Duration::from_micros(1), + }]; + + let report = build_report(&Driver::Sqlite3, &db_version, 1, timings_ms, operation_stats); + + assert_eq!(report.meta.driver, "sqlite3"); + assert_eq!(report.meta.db_version, "-"); + } + + #[test] + fn it_should_keep_mysql_db_version_in_report_metadata() { + let db_version = DbVersion::from_str("8.4").expect("db version should parse"); + let timings_ms = ReportTimings { + benchmark: 9, + report_build: 1, + total: 10, + }; + let operation_stats = vec![OperationStats { + name: "load_keys".to_string(), + count: 2, + best: Duration::from_micros(2), + median: Duration::from_micros(3), + worst: Duration::from_micros(4), + }]; + + let report = build_report(&Driver::MySQL, &db_version, 2, timings_ms, operation_stats); + + assert_eq!(report.meta.driver, "mysql"); + assert_eq!(report.meta.db_version, "8.4"); + assert_eq!(report.meta.ops, 2); + } +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/runner.rs b/packages/tracker-core/src/bin/persistence_benchmark/runner.rs new file mode 100644 index 000000000..81d871a6c --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/runner.rs @@ -0,0 +1,71 @@ +use std::time::Instant; + +use anyhow::Result; +use bittorrent_tracker_core::databases::driver::Driver; +use clap::Parser; + +use super::types::{DbVersion, OpsCount}; +use super::{operations, report, reporting}; + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + /// Database driver benchmarked in this invocation. + #[arg(long)] + driver: Driver, + + /// Database image tag. Used only for `MySQL`. + #[arg(long, default_value = "8.4")] + db_version: DbVersion, + + /// Number of samples per operation. + #[arg(long, default_value = "100")] + ops: OpsCount, +} + +/// Executes the persistence benchmark runner CLI. +/// +/// # Errors +/// +/// Returns an error if argument validation fails, the benchmark execution +/// fails, or report serialization fails. +pub async fn run() -> Result<()> { + let Args { driver, db_version, ops } = Args::parse(); + + let total_started_at = Instant::now(); + + let benchmark_started_at = Instant::now(); + let operation_stats = operations::collect_operation_stats(&driver, &db_version, ops).await?; + let benchmark_duration = benchmark_started_at.elapsed(); + + let report_build_started_at = Instant::now(); + let mut benchmark_report = reporting::build_report( + &driver, + &db_version, + ops.get(), + report::ReportTimings { + benchmark: 0, + report_build: 0, + total: 0, + }, + operation_stats, + ); + let report_build_duration = report_build_started_at.elapsed(); + + let total_duration = total_started_at.elapsed(); + benchmark_report.meta.timings_ms = report::ReportTimings { + benchmark: duration_to_millis_u64(benchmark_duration), + report_build: duration_to_millis_u64(report_build_duration), + total: duration_to_millis_u64(total_duration), + }; + + let json = report::to_json_pretty(&benchmark_report)?; + + println!("{json}"); + + Ok(()) +} + +fn duration_to_millis_u64(duration: std::time::Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/types.rs b/packages/tracker-core/src/bin/persistence_benchmark/types.rs new file mode 100644 index 000000000..15a3b36cf --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/types.rs @@ -0,0 +1,114 @@ +use std::num::NonZeroUsize; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OpsCount(NonZeroUsize); + +impl OpsCount { + #[must_use] + pub fn get(self) -> usize { + self.0.get() + } +} + +impl FromStr for OpsCount { + type Err = String; + + fn from_str(value: &str) -> Result { + let parsed = value + .parse::() + .map_err(|_| "ops must be a positive integer".to_string())?; + + let count = NonZeroUsize::new(parsed).ok_or_else(|| "ops must be greater than zero".to_string())?; + + Ok(Self(count)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DbVersion(String); + +impl DbVersion { + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for DbVersion { + type Err = String; + + fn from_str(value: &str) -> Result { + if value.is_empty() { + return Err("db-version must not be empty".to_string()); + } + + let is_valid = value + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_')); + + if !is_valid { + return Err("db-version contains invalid characters; allowed: letters, digits, '.', '-', '_'".to_string()); + } + + Ok(Self(value.to_string())) + } +} + +impl std::fmt::Display for DbVersion { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::{DbVersion, OpsCount}; + + #[test] + fn it_should_parse_ops_count_when_value_is_positive() { + let ops = OpsCount::from_str("100").expect("ops count should parse"); + + assert_eq!(ops.get(), 100); + } + + #[test] + fn it_should_reject_ops_count_when_value_is_zero() { + let error = OpsCount::from_str("0").expect_err("zero ops count should fail"); + + assert_eq!(error, "ops must be greater than zero"); + } + + #[test] + fn it_should_reject_ops_count_when_value_is_not_numeric() { + let error = OpsCount::from_str("abc").expect_err("non-numeric ops count should fail"); + + assert_eq!(error, "ops must be a positive integer"); + } + + #[test] + fn it_should_parse_db_version_when_value_has_allowed_characters() { + let db_version = DbVersion::from_str("8.4-rc1").expect("db version should parse"); + + assert_eq!(db_version.as_str(), "8.4-rc1"); + } + + #[test] + fn it_should_reject_db_version_when_value_is_empty() { + let error = DbVersion::from_str("").expect_err("empty db version should fail"); + + assert_eq!(error, "db-version must not be empty"); + } + + #[test] + fn it_should_reject_db_version_when_value_has_invalid_characters() { + let error = DbVersion::from_str("8.4/rc1").expect_err("db version with slash should fail"); + + assert_eq!( + error, + "db-version contains invalid characters; allowed: letters, digits, '.', '-', '_'" + ); + } +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark_runner.rs b/packages/tracker-core/src/bin/persistence_benchmark_runner.rs new file mode 100644 index 000000000..357443a23 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark_runner.rs @@ -0,0 +1,76 @@ +//! Program to run persistence benchmarks directly against database drivers. +//! +//! This binary is a developer tool for measuring the persistence-layer methods +//! implemented by the [`Database`](bittorrent_tracker_core::databases::Database) +//! trait. It benchmarks one driver per invocation and prints a JSON report to +//! standard output with per-operation timing statistics. +//! +//! How it works: +//! +//! - Parses CLI arguments for the target driver, database version, and sample +//! count (`--ops`, default: `100`). +//! - Instantiates a real persistence backend: +//! - `sqlite3` uses a temporary `SQLite` database file. +//! - `mysql` starts a testcontainers `mysql` container with the requested +//! image tag. +//! - Creates a clean schema and seeds the minimum data needed for each measured +//! operation. +//! - Repeats every persistence operation `--ops` times, measuring each call +//! with `std::time::Instant`. +//! - Sorts the collected durations and prints `count`, `best`, `median`, and +//! `worst` values as JSON. +//! - Emits only JSON on standard output (no status line and no file output +//! argument). +//! +//! Typical usage: +//! +//! ```text +//! cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ +//! --driver sqlite3 +//! +//! cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ +//! --driver mysql \ +//! --db-version 8.4 +//! ``` +//! +//! Store output in a file with shell redirection: +//! +//! ```text +//! cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ +//! --driver sqlite3 \ +//! > .benchmarks/bench-results-sqlite3.json +//! ``` +//! +//! Sample report: +//! +//! ```json +//! { +//! "meta": { +//! "git_revision": "16c9c8a4695d336a4531204913390a47b20d9468", +//! "driver": "sqlite3", +//! "db_version": "-", +//! "ops": 100, +//! "timestamp": "2026-04-28T16:23:24.084307218+00:00", +//! "timings_ms": { +//! "benchmark": 18, +//! "report_build": 0, +//! "total": 19 +//! } +//! }, +//! "operations": [ +//! { +//! "name": "save_torrent_downloads", +//! "count": 100, +//! "best_us": 66, +//! "median_us": 70, +//! "worst_us": 79 +//! } +//! ] +//! } +//! ``` +mod persistence_benchmark; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + persistence_benchmark::runner::run().await +} diff --git a/packages/tracker-core/src/databases/driver/mod.rs b/packages/tracker-core/src/databases/driver/mod.rs index 6c849bb70..7126e2e98 100644 --- a/packages/tracker-core/src/databases/driver/mod.rs +++ b/packages/tracker-core/src/databases/driver/mod.rs @@ -1,4 +1,6 @@ //! Database driver factory. +use std::str::FromStr; + use mysql::Mysql; use serde::{Deserialize, Serialize}; use sqlite::Sqlite; @@ -25,6 +27,29 @@ pub enum Driver { MySQL, } +impl Driver { + /// Returns the stable lowercase identifier used by CLI and reports. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Sqlite3 => "sqlite3", + Self::MySQL => "mysql", + } + } +} + +impl FromStr for Driver { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "sqlite3" => Ok(Self::Sqlite3), + "mysql" => Ok(Self::MySQL), + _ => Err("driver must be one of: sqlite3, mysql".to_string()), + } + } +} + /// It builds a new database driver. /// /// Example for `SQLite3`: From 505b8329daaaee8cf7d080e344e22cda60d8d848 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Tue, 28 Apr 2026 20:16:59 +0100 Subject: [PATCH 4/5] docs(tracker-core): add persistence benchmark baseline artifacts Add the 2026-04-28 baseline benchmarking docs, machine profile, and raw sqlite/mysql JSON results under tracker-core docs. Update cspell ignore patterns for benchmark machine artifacts as a lint follow-up. --- cspell.json | 3 +- .../tracker-core/docs/benchmarking/README.md | 65 ++++++++++ .../machine/2026-04-28-josecelano-desktop.txt | 94 ++++++++++++++ .../benchmarking/runs/2026-04-28/REPORT.md | 66 ++++++++++ .../runs/2026-04-28/mysql-8.0.json | 121 ++++++++++++++++++ .../runs/2026-04-28/mysql-8.4.json | 121 ++++++++++++++++++ .../benchmarking/runs/2026-04-28/sqlite3.json | 121 ++++++++++++++++++ 7 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 packages/tracker-core/docs/benchmarking/README.md create mode 100644 packages/tracker-core/docs/benchmarking/machine/2026-04-28-josecelano-desktop.txt create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-04-28/REPORT.md create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.0.json create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.4.json create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-04-28/sqlite3.json diff --git a/cspell.json b/cspell.json index af6245e65..876291c36 100644 --- a/cspell.json +++ b/cspell.json @@ -21,9 +21,10 @@ "docs/media/*.svg", "contrib/bencode/benches/*.bencode", "contrib/dev-tools/su-exec/**", + "packages/tracker-core/docs/benchmarking/machine/*.txt", ".github/labels.json", "/project-words.txt", "repomix-output.xml", "TEMP-*.md" ] -} +} \ No newline at end of file diff --git a/packages/tracker-core/docs/benchmarking/README.md b/packages/tracker-core/docs/benchmarking/README.md new file mode 100644 index 000000000..e8fac458a --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/README.md @@ -0,0 +1,65 @@ +# Persistence Benchmarking Reports + +This folder stores benchmark artifacts produced by +`persistence_benchmark_runner` for `bittorrent-tracker-core`. + +Goals: + +- Keep reproducible baseline reports in-repo. +- Track benchmark evolution across major persistence changes. +- Enable before/after comparisons (for example, before and after SQLx migration). + +## Layout + +- `machine/`: machine and toolchain characteristics for each run date. +- `runs//`: raw JSON benchmark output files and a run summary. + +## Baseline run (pre-SQLx) + +- Date: `2026-04-28` +- Commit: `51c27fda813876afc1cb26ea1d5bbb0fa49dfdd2` +- Issue context: `docs/issues/1710-1525-03-persistence-benchmarking.md` +- Run summary: `runs/2026-04-28/REPORT.md` +- Machine profile: `machine/2026-04-28-josecelano-desktop.txt` + +Raw JSON artifacts: + +- `runs/2026-04-28/sqlite3.json` +- `runs/2026-04-28/mysql-8.4.json` +- `runs/2026-04-28/mysql-8.0.json` + +## How to add a new run + +1. Create a new run folder: + + `mkdir -p packages/tracker-core/docs/benchmarking/runs/YYYY-MM-DD` + +2. Run benchmarks and save JSON artifacts: + + `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver sqlite3 > packages/tracker-core/docs/benchmarking/runs/YYYY-MM-DD/sqlite3.json` + + `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4 > packages/tracker-core/docs/benchmarking/runs/YYYY-MM-DD/mysql-8.4.json` + +3. Capture machine profile: + + `mkdir -p packages/tracker-core/docs/benchmarking/machine` + + Save at least OS, kernel, CPU, RAM, Rust toolchain and container runtime versions to: + + `packages/tracker-core/docs/benchmarking/machine/YYYY-MM-DD-.txt` + +4. Add `runs/YYYY-MM-DD/REPORT.md` with: + - benchmark context (commit, command, ops) + - high-level summary (total benchmark time) + - important per-operation medians + - comparison versus a prior run when relevant + +5. Update this index file with links to the new run and machine profile. + +## Planned comparison point + +After implementing: + +- `docs/issues/1525-05-migrate-sqlite-and-mysql-to-sqlx.md` + +run the same benchmark commands again, store results in a new dated folder, and compare against `runs/2026-04-28`. diff --git a/packages/tracker-core/docs/benchmarking/machine/2026-04-28-josecelano-desktop.txt b/packages/tracker-core/docs/benchmarking/machine/2026-04-28-josecelano-desktop.txt new file mode 100644 index 000000000..9a3d20f31 --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/machine/2026-04-28-josecelano-desktop.txt @@ -0,0 +1,94 @@ +hostname: +josecelano-desktop + +date_utc: +2026-04-28T18:40:06Z + +uname -a: +Linux josecelano-desktop 6.17.0-22-generic #22-Ubuntu SMP PREEMPT_DYNAMIC Fri Mar 13 12:04:44 UTC 2026 x86_64 GNU/Linux + +/etc/os-release: +PRETTY_NAME="Ubuntu 25.10" +NAME="Ubuntu" +VERSION_ID="25.10" +VERSION="25.10 (Questing Quokka)" +VERSION_CODENAME=questing +ID=ubuntu +ID_LIKE=debian +HOME_URL="https://www.ubuntu.com/" +SUPPORT_URL="https://help.ubuntu.com/" +BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" +PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" +UBUNTU_CODENAME=questing +LOGO=ubuntu-logo + +lscpu: +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Address sizes: 48 bits physical, 48 bits virtual +Byte Order: Little Endian +CPU(s): 32 +On-line CPU(s) list: 0-31 +Vendor ID: AuthenticAMD +Model name: AMD Ryzen 9 7950X 16-Core Processor +CPU family: 25 +Model: 97 +Thread(s) per core: 2 +Core(s) per socket: 16 +Socket(s): 1 +Stepping: 2 +Frequency boost: enabled +CPU(s) scaling MHz: 76% +CPU max MHz: 5883,1968 +CPU min MHz: 425,2920 +BogoMIPS: 8982,52 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpuid_fault cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d amd_lbr_pmc_freeze +Virtualization: AMD-V +L1d cache: 512 KiB (16 instances) +L1i cache: 512 KiB (16 instances) +L2 cache: 16 MiB (16 instances) +L3 cache: 64 MiB (2 instances) +NUMA node(s): 1 +NUMA node0 CPU(s): 0-31 +Vulnerability Gather data sampling: Not affected +Vulnerability Ghostwrite: Not affected +Vulnerability Indirect target selection: Not affected +Vulnerability Itlb multihit: Not affected +Vulnerability L1tf: Not affected +Vulnerability Mds: Not affected +Vulnerability Meltdown: Not affected +Vulnerability Mmio stale data: Not affected +Vulnerability Old microcode: Not affected +Vulnerability Reg file data sampling: Not affected +Vulnerability Retbleed: Not affected +Vulnerability Spec rstack overflow: Mitigation; Safe RET +Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl +Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization +Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; STIBP always-on; PBRSB-eIBRS Not affected; BHI Not affected +Vulnerability Srbds: Not affected +Vulnerability Tsa: Mitigation; Clear CPU buffers +Vulnerability Tsx async abort: Not affected +Vulnerability Vmscape: Mitigation; IBPB before exit to userspace + +free -h: + total used free shared buff/cache available +Mem: 61Gi 21Gi 24Gi 589Mi 16Gi 39Gi +Swap: 8,0Gi 2,4Gi 5,6Gi + +rustc -Vv: +rustc 1.97.0-nightly (52b6e2c20 2026-04-27) +binary: rustc +commit-hash: 52b6e2c208b73276ccb36ec0b68456913a801c96 +commit-date: 2026-04-27 +host: x86_64-unknown-linux-gnu +release: 1.97.0-nightly +LLVM version: 22.1.2 + +cargo -V: +cargo 1.97.0-nightly (eb9b60f1f 2026-04-24) + +docker version: +28.3.3 + +podman version: +podman-not-available diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-04-28/REPORT.md b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/REPORT.md new file mode 100644 index 000000000..8df135c0d --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/REPORT.md @@ -0,0 +1,66 @@ +# Benchmark Report - 2026-04-28 + +This is the baseline benchmark run captured after implementing: + +- `docs/issues/1710-1525-03-persistence-benchmarking.md` + +## Run context + +- Commit: `51c27fda813876afc1cb26ea1d5bbb0fa49dfdd2` +- Ops per operation: `100` +- Benchmark runner: `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner` +- Machine profile: `../../machine/2026-04-28-josecelano-desktop.txt` + +## Raw artifacts + +- `sqlite3.json` +- `mysql-8.4.json` +- `mysql-8.0.json` + +## High-level timing summary + +`meta.timings_ms.total`: + +- sqlite3: `75 ms` +- mysql 8.4: `7381 ms` +- mysql 8.0: `7633 ms` + +Interpretation: + +- sqlite3 is much faster on this local setup. +- mysql 8.4 is slightly faster than mysql 8.0 in this run set. + +## Selected operation medians (microseconds) + +| Operation | sqlite3 | mysql 8.4 | mysql 8.0 | +| ------------------------------- | ------: | --------: | --------: | +| save_torrent_downloads | 64 | 750 | 949 | +| load_torrent_downloads | 9 | 114 | 133 | +| increase_downloads_for_torrent | 50 | 759 | 1027 | +| save_global_downloads | 58 | 745 | 1020 | +| increase_global_downloads | 49 | 748 | 1007 | +| add_info_hash_to_whitelist | 61 | 715 | 998 | +| remove_info_hash_from_whitelist | 116 | 1460 | 1902 | +| add_key_to_keys | 61 | 712 | 948 | +| remove_key_from_keys | 116 | 1476 | 1883 | + +## Machine characteristics (summary) + +From `../../machine/2026-04-28-josecelano-desktop.txt`: + +- Host: `josecelano-desktop` +- OS: `Ubuntu 25.10` +- Kernel: `Linux 6.17.0-22-generic` +- CPU: `AMD Ryzen 9 7950X` (16 cores / 32 threads) +- RAM: `61 GiB` +- Rust: `rustc 1.97.0-nightly (LLVM 22.1.2)` +- Cargo: `1.97.0-nightly` +- Container runtime used by benchmark: `Docker 28.3.3` + +## Next comparison milestone + +After implementing: + +- `docs/issues/1525-05-migrate-sqlite-and-mysql-to-sqlx.md` + +run the same commands, store results under a new date folder, and compare medians and totals against this baseline. diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.0.json b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.0.json new file mode 100644 index 000000000..5955da33c --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.0.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "51c27fda813876afc1cb26ea1d5bbb0fa49dfdd2", + "driver": "mysql", + "db_version": "8.0", + "ops": 100, + "timestamp": "2026-04-28T18:37:46.176977790+00:00", + "timings_ms": { + "benchmark": 7632, + "report_build": 1, + "total": 7633 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 725, + "median_us": 949, + "worst_us": 1778 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 117, + "median_us": 133, + "worst_us": 474 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 155, + "median_us": 160, + "worst_us": 254 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 928, + "median_us": 1027, + "worst_us": 1463 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 738, + "median_us": 1020, + "worst_us": 1570 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 115, + "median_us": 117, + "worst_us": 267 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 741, + "median_us": 1007, + "worst_us": 1493 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 702, + "median_us": 998, + "worst_us": 1491 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 115, + "median_us": 118, + "worst_us": 295 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 149, + "median_us": 151, + "worst_us": 203 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 1642, + "median_us": 1902, + "worst_us": 2519 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 714, + "median_us": 948, + "worst_us": 1317 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 129, + "median_us": 131, + "worst_us": 317 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 161, + "median_us": 180, + "worst_us": 266 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 1631, + "median_us": 1883, + "worst_us": 4593 + } + ] +} diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.4.json b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.4.json new file mode 100644 index 000000000..f403d036c --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/mysql-8.4.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "51c27fda813876afc1cb26ea1d5bbb0fa49dfdd2", + "driver": "mysql", + "db_version": "8.4", + "ops": 100, + "timestamp": "2026-04-28T18:39:26.804522153+00:00", + "timings_ms": { + "benchmark": 7380, + "report_build": 1, + "total": 7381 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 695, + "median_us": 750, + "worst_us": 3000 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 109, + "median_us": 114, + "worst_us": 253 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 142, + "median_us": 146, + "worst_us": 225 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 712, + "median_us": 759, + "worst_us": 1248 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 692, + "median_us": 745, + "worst_us": 1453 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 107, + "median_us": 117, + "worst_us": 243 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 694, + "median_us": 748, + "worst_us": 1178 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 688, + "median_us": 715, + "worst_us": 1556 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 108, + "median_us": 110, + "worst_us": 233 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 147, + "median_us": 150, + "worst_us": 228 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 1400, + "median_us": 1460, + "worst_us": 1935 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 689, + "median_us": 712, + "worst_us": 1113 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 108, + "median_us": 110, + "worst_us": 252 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 155, + "median_us": 174, + "worst_us": 246 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 1402, + "median_us": 1476, + "worst_us": 2181 + } + ] +} diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-04-28/sqlite3.json b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/sqlite3.json new file mode 100644 index 000000000..ee792a961 --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-04-28/sqlite3.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "51c27fda813876afc1cb26ea1d5bbb0fa49dfdd2", + "driver": "sqlite3", + "db_version": "-", + "ops": 100, + "timestamp": "2026-04-28T18:37:30.676323598+00:00", + "timings_ms": { + "benchmark": 73, + "report_build": 1, + "total": 75 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 62, + "median_us": 64, + "worst_us": 73 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 9, + "median_us": 9, + "worst_us": 17 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 24, + "median_us": 24, + "worst_us": 36 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 48, + "median_us": 50, + "worst_us": 64 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 57, + "median_us": 58, + "worst_us": 194 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 8, + "median_us": 9, + "worst_us": 16 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 48, + "median_us": 49, + "worst_us": 191 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 60, + "median_us": 61, + "worst_us": 75 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 8, + "median_us": 9, + "worst_us": 220 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 18, + "median_us": 18, + "worst_us": 30 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 114, + "median_us": 116, + "worst_us": 375 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 59, + "median_us": 61, + "worst_us": 344 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 9, + "median_us": 9, + "worst_us": 16 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 25, + "median_us": 25, + "worst_us": 46 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 113, + "median_us": 116, + "worst_us": 384 + } + ] +} From 56478904d83e0641a24c9720ca4ed4a722a71f0d Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Tue, 28 Apr 2026 21:27:15 +0100 Subject: [PATCH 5/5] refactor(tracker-core): address Copilot PR review suggestions - Separate fixture setup from timed section in measure_operation by switching to a two-closure (setup, operation) signature so recorded durations reflect only the database call. - Move database handle into Option>> so it is explicitly dropped before the SQLite file is removed in Drop. - Preserve the last error from create_database_tables_with_retry instead of discarding it, making container startup failures easier to diagnose. - Align docs/issues/1710-1525-03-persistence-benchmarking.md with the implementation: ops default 100, stdout-only JSON output, and correct artifact paths under packages/tracker-core/docs/benchmarking. --- .../1710-1525-03-persistence-benchmarking.md | 34 +++--- .../driver_bench/database/mod.rs | 19 ++- .../driver_bench/database/mysql.rs | 2 +- .../driver_bench/database/sqlite.rs | 2 +- .../persistence_benchmark/driver_bench/mod.rs | 9 +- .../driver_bench/operations/keys.rs | 73 +++++++----- .../driver_bench/operations/torrent.rs | 112 +++++++++++------- .../driver_bench/operations/whitelist.rs | 77 +++++++----- .../driver_bench/sampling.rs | 19 ++- 9 files changed, 219 insertions(+), 128 deletions(-) diff --git a/docs/issues/1710-1525-03-persistence-benchmarking.md b/docs/issues/1710-1525-03-persistence-benchmarking.md index 690ef75cd..2da0a7e8b 100644 --- a/docs/issues/1710-1525-03-persistence-benchmarking.md +++ b/docs/issues/1710-1525-03-persistence-benchmarking.md @@ -24,9 +24,9 @@ already covered by tests, otherwise performance comparisons risk masking regress must be designed so PostgreSQL can be added in subissue #1525-08 without redesign. - One invocation produces results for one driver/version combination. Run it three times to cover `sqlite3`, `mysql:8.0`, and `mysql:8.4`. -- Commit one JSON report per combination under `docs/benchmarks/` as the baseline. Re-run - and update the reports in each subsequent subissue that changes persistence behavior. The - git diff of those JSON files is the before/after comparison. +- Commit one JSON report per combination under `packages/tracker-core/docs/benchmarking/runs/` + as the baseline. Re-run and update the reports in each subsequent subissue that changes + persistence behavior. The git diff of those JSON files is the before/after comparison. ## Measurement Tool Rationale @@ -55,10 +55,10 @@ Every method on the `Database` trait, grouped by category: | Whitelist | `add_info_hash_to_whitelist`, `get_info_hash_from_whitelist`, `load_whitelist`, `remove_info_hash_from_whitelist` | | Auth keys | `add_key_to_keys`, `get_key_from_keys`, `load_keys`, `remove_key_from_keys` | -Each method is called `--ops N` times (default `10`). The collected `Vec` is sorted +Each method is called `--ops N` times (default `100`). The collected `Vec` is sorted to produce `count`, `best`, `median`, and `worst` per operation. -A default of `10` is deliberately small so a local run finishes well under 3 minutes. +A default of `100` matches the committed baseline reports and produces stable medians. Pass a larger `--ops` value when tighter statistics are needed. ## What Is NOT Measured @@ -134,8 +134,8 @@ Run `cargo machete` after to verify no unused dependencies remain. cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ --driver sqlite3|mysql # exactly one driver per run --db-version 8.4 # DB image tag; ignored for sqlite3; default "8.4" for mysql - --ops 10 # samples per operation; default 10 - --json-output # default: .benchmarks/bench-results-[-].json + --ops 100 # samples per operation; default 100 + # JSON report is printed to stdout; redirect to save it ``` **Driver setup:** @@ -160,7 +160,7 @@ cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ "git_revision": "", "driver": "sqlite3", "db_version": "-", - "ops": 10, + "ops": 100, "timestamp": "2026-04-28T12:00:00Z" }, "operations": [ @@ -178,9 +178,9 @@ cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ Acceptance criteria: - [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver sqlite3` - runs to completion and writes a JSON report. + runs to completion and prints a JSON report to stdout. - [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4` - runs to completion and writes a JSON report. + runs to completion and prints a JSON report to stdout. - [ ] JSON schema matches the structure above. - [ ] `cargo machete` reports no unused dependencies. @@ -193,21 +193,21 @@ reports alongside the code change. The git diff is the before/after comparison. ```bash cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ --driver sqlite3 \ - --json-output docs/benchmarks/baseline-sqlite3.json + > packages/tracker-core/docs/benchmarking/runs/$(date +%F)/sqlite3.json cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ --driver mysql --db-version 8.0 \ - --json-output docs/benchmarks/baseline-mysql-8.0.json + > packages/tracker-core/docs/benchmarking/runs/$(date +%F)/mysql-8.0.json cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- \ --driver mysql --db-version 8.4 \ - --json-output docs/benchmarks/baseline-mysql-8.4.json + > packages/tracker-core/docs/benchmarking/runs/$(date +%F)/mysql-8.4.json ``` Acceptance criteria: -- [ ] `docs/benchmarks/baseline-sqlite3.json`, `docs/benchmarks/baseline-mysql-8.0.json`, - and `docs/benchmarks/baseline-mysql-8.4.json` are committed. +- [ ] `packages/tracker-core/docs/benchmarking/runs//sqlite3.json`, + `mysql-8.0.json`, and `mysql-8.4.json` are committed. - [ ] Each file identifies the git revision, driver, db-version, ops count, and timestamp. ### 3) Document the workflow @@ -239,8 +239,8 @@ Acceptance criteria: runs to completion and prints a summary. - [ ] `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4` runs to completion and prints a summary. -- [ ] `docs/benchmarks/baseline-sqlite3.json`, `docs/benchmarks/baseline-mysql-8.0.json`, - and `docs/benchmarks/baseline-mysql-8.4.json` are committed. +- [ ] `packages/tracker-core/docs/benchmarking/runs//sqlite3.json`, + `mysql-8.0.json`, and `mysql-8.4.json` are committed. - [ ] `docs/benchmarking.md` documents the workflow. - [ ] `cargo test --workspace --all-targets` passes. - [ ] `linter all` exits with code `0`. diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs index 70f8142d5..1656b2303 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs @@ -11,7 +11,7 @@ mod mysql; mod sqlite; pub(super) struct ActiveDatabase { - pub(super) database: Arc>, + pub(super) database: Option>>, resource: Option, } @@ -41,6 +41,9 @@ impl ActiveDatabase { impl Drop for ActiveDatabase { fn drop(&mut self) { + // Drop the database connection before cleaning up the resource. + // For SQLite this ensures the file handle is released before removal. + drop(self.database.take()); match self.resource.take() { Some(BenchmarkResource::Sqlite(path)) => { let _removed_file_result = std::fs::remove_file(path); @@ -70,13 +73,21 @@ pub(super) async fn reset_database(database: &dyn Database) -> Result<()> { /// /// Returns an error if the database is still not ready after all retries. async fn create_database_tables_with_retry(database: &dyn Database) -> Result<()> { + let mut last_error: Option = None; + for _ in 0..5 { - if database.create_database_tables().is_ok() { - return Ok(()); + match database.create_database_tables() { + Ok(()) => return Ok(()), + Err(error) => { + last_error = Some(error.into()); + } } tokio::time::sleep(Duration::from_secs(2)).await; } - Err(anyhow!("database is not ready after retries")) + match last_error { + Some(error) => Err(anyhow!("database is not ready after retries; last error: {error}")), + None => Err(anyhow!("database is not ready after retries")), + } } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs index 3caad237f..4bbc332c7 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mysql.rs @@ -33,7 +33,7 @@ pub(super) async fn initialize(db_version: &str) -> Result { let database = initialize_database(&config); Ok(ActiveDatabase { - database, + database: Some(database), resource: Some(BenchmarkResource::Mysql(Box::new(mysql_container))), }) } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs index f597cc32b..1ffa06198 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/sqlite.rs @@ -16,7 +16,7 @@ pub(super) fn initialize() -> ActiveDatabase { let database = initialize_database(&config); ActiveDatabase { - database, + database: Some(database), resource: Some(BenchmarkResource::Sqlite(sqlite_db_path)), } } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs index 674eb3428..a91fbbc56 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/mod.rs @@ -23,14 +23,15 @@ pub struct RawOperationSamples { /// operation fails. pub async fn run(driver: Driver, db_version: &str, ops: OpsCount) -> Result> { let active_database = database::ActiveDatabase::new(driver, db_version).await?; - database::reset_database(active_database.database.as_ref().as_ref()).await?; + let db = active_database.database.as_deref().unwrap().as_ref(); + database::reset_database(db).await?; let ops = ops.get(); let mut operations_samples = Vec::new(); - operations::benchmark_torrent_operations(active_database.database.as_ref().as_ref(), ops, &mut operations_samples)?; - operations::benchmark_whitelist_operations(active_database.database.as_ref().as_ref(), ops, &mut operations_samples)?; - operations::benchmark_key_operations(active_database.database.as_ref().as_ref(), ops, &mut operations_samples)?; + operations::benchmark_torrent_operations(db, ops, &mut operations_samples)?; + operations::benchmark_whitelist_operations(db, ops, &mut operations_samples)?; + operations::benchmark_key_operations(db, ops, &mut operations_samples)?; Ok(operations_samples) } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs index 388147cc2..484640784 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/keys.rs @@ -15,41 +15,60 @@ pub(super) fn benchmark_key_operations( ops: usize, operations: &mut Vec, ) -> Result<()> { - operations.push(measure_operation("add_key_to_keys", ops, |_| { - let peer_key = authentication::key::generate_key(None); - let _added_rows = database.add_key_to_keys(&peer_key).context("add_key_to_keys failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "add_key_to_keys", + ops, + |_| Ok(authentication::key::generate_key(None)), + |peer_key| { + let _added_rows = database.add_key_to_keys(&peer_key).context("add_key_to_keys failed")?; + Ok(()) + }, + )?); let persisted_peer_key = authentication::key::generate_key(None); let _added_rows = database .add_key_to_keys(&persisted_peer_key) .context("failed to seed get_key_from_keys")?; let persisted_key = persisted_peer_key.key(); - operations.push(measure_operation("get_key_from_keys", ops, |_| { - let persisted_key_result = database - .get_key_from_keys(&persisted_key) - .context("get_key_from_keys failed")?; - drop(persisted_key_result); - Ok(()) - })?); + operations.push(measure_operation( + "get_key_from_keys", + ops, + |_| Ok(()), + |()| { + let persisted_key_result = database + .get_key_from_keys(&persisted_key) + .context("get_key_from_keys failed")?; + drop(persisted_key_result); + Ok(()) + }, + )?); - operations.push(measure_operation("load_keys", ops, |_| { - let keys = database.load_keys().context("load_keys failed")?; - drop(keys); - Ok(()) - })?); + operations.push(measure_operation( + "load_keys", + ops, + |_| Ok(()), + |()| { + let keys = database.load_keys().context("load_keys failed")?; + drop(keys); + Ok(()) + }, + )?); - operations.push(measure_operation("remove_key_from_keys", ops, |_| { - let peer_key = authentication::key::generate_key(None); - let _added_rows = database - .add_key_to_keys(&peer_key) - .context("failed to seed remove_key_from_keys")?; - let _removed_rows = database - .remove_key_from_keys(&peer_key.key()) - .context("remove_key_from_keys failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "remove_key_from_keys", + ops, + |_| { + let peer_key = authentication::key::generate_key(None); + let _added_rows = database + .add_key_to_keys(&peer_key) + .context("failed to seed remove_key_from_keys")?; + Ok(peer_key.key()) + }, + |key| { + let _removed_rows = database.remove_key_from_keys(&key).context("remove_key_from_keys failed")?; + Ok(()) + }, + )?); Ok(()) } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs index ca7fb28b2..993a60c74 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/torrent.rs @@ -17,66 +17,98 @@ pub(super) fn benchmark_torrent_operations( ops: usize, operations: &mut Vec, ) -> Result<()> { - operations.push(measure_operation("save_torrent_downloads", ops, |index| { - let info_hash = info_hash_from_index(index + 1)?; - let downloads = downloads_from_index(index)?; - database - .save_torrent_downloads(&info_hash, downloads) - .context("save_torrent_downloads failed") - })?); + operations.push(measure_operation( + "save_torrent_downloads", + ops, + |index| Ok((info_hash_from_index(index + 1)?, downloads_from_index(index)?)), + |(info_hash, downloads)| { + database + .save_torrent_downloads(&info_hash, downloads) + .context("save_torrent_downloads failed") + }, + )?); let load_torrent_info_hash = info_hash_from_index(10_000)?; database .save_torrent_downloads(&load_torrent_info_hash, 123) .context("failed to seed load_torrent_downloads")?; - operations.push(measure_operation("load_torrent_downloads", ops, |_| { - let _downloads_result = database - .load_torrent_downloads(&load_torrent_info_hash) - .context("load_torrent_downloads failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "load_torrent_downloads", + ops, + |_| Ok(()), + |()| { + let _downloads_result = database + .load_torrent_downloads(&load_torrent_info_hash) + .context("load_torrent_downloads failed")?; + Ok(()) + }, + )?); - operations.push(measure_operation("load_all_torrents_downloads", ops, |_| { - let all_downloads = database - .load_all_torrents_downloads() - .context("load_all_torrents_downloads failed")?; - drop(all_downloads); - Ok(()) - })?); + operations.push(measure_operation( + "load_all_torrents_downloads", + ops, + |_| Ok(()), + |()| { + let all_downloads = database + .load_all_torrents_downloads() + .context("load_all_torrents_downloads failed")?; + drop(all_downloads); + Ok(()) + }, + )?); let increasing_downloads_info_hash = info_hash_from_index(20_000)?; database .save_torrent_downloads(&increasing_downloads_info_hash, 0) .context("failed to seed increase_downloads_for_torrent")?; - operations.push(measure_operation("increase_downloads_for_torrent", ops, |_| { - database - .increase_downloads_for_torrent(&increasing_downloads_info_hash) - .context("increase_downloads_for_torrent failed") - })?); + operations.push(measure_operation( + "increase_downloads_for_torrent", + ops, + |_| Ok(()), + |()| { + database + .increase_downloads_for_torrent(&increasing_downloads_info_hash) + .context("increase_downloads_for_torrent failed") + }, + )?); - operations.push(measure_operation("save_global_downloads", ops, |index| { - let downloads = downloads_from_index(index)?; - database - .save_global_downloads(downloads) - .context("save_global_downloads failed") - })?); + operations.push(measure_operation( + "save_global_downloads", + ops, + downloads_from_index, + |downloads| { + database + .save_global_downloads(downloads) + .context("save_global_downloads failed") + }, + )?); database .save_global_downloads(0) .context("failed to seed load_global_downloads")?; - operations.push(measure_operation("load_global_downloads", ops, |_| { - let _downloads_result = database.load_global_downloads().context("load_global_downloads failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "load_global_downloads", + ops, + |_| Ok(()), + |()| { + let _downloads_result = database.load_global_downloads().context("load_global_downloads failed")?; + Ok(()) + }, + )?); database .save_global_downloads(0) .context("failed to seed increase_global_downloads")?; - operations.push(measure_operation("increase_global_downloads", ops, |_| { - database - .increase_global_downloads() - .context("increase_global_downloads failed") - })?); + operations.push(measure_operation( + "increase_global_downloads", + ops, + |_| Ok(()), + |()| { + database + .increase_global_downloads() + .context("increase_global_downloads failed") + }, + )?); Ok(()) } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs index 2efb25cb9..2c5b8366e 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/operations/whitelist.rs @@ -14,41 +14,62 @@ pub(super) fn benchmark_whitelist_operations( ops: usize, operations: &mut Vec, ) -> Result<()> { - operations.push(measure_operation("add_info_hash_to_whitelist", ops, |index| { - let info_hash = info_hash_from_index(30_000 + index)?; - let _added_rows = database - .add_info_hash_to_whitelist(info_hash) - .context("add_info_hash_to_whitelist failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "add_info_hash_to_whitelist", + ops, + |index| info_hash_from_index(30_000 + index), + |info_hash| { + let _added_rows = database + .add_info_hash_to_whitelist(info_hash) + .context("add_info_hash_to_whitelist failed")?; + Ok(()) + }, + )?); let whitelisted_info_hash = info_hash_from_index(40_000)?; let _added_rows = database .add_info_hash_to_whitelist(whitelisted_info_hash) .context("failed to seed get_info_hash_from_whitelist")?; - operations.push(measure_operation("get_info_hash_from_whitelist", ops, |_| { - let _info_hash_result = database - .get_info_hash_from_whitelist(whitelisted_info_hash) - .context("get_info_hash_from_whitelist failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "get_info_hash_from_whitelist", + ops, + |_| Ok(()), + |()| { + let _info_hash_result = database + .get_info_hash_from_whitelist(whitelisted_info_hash) + .context("get_info_hash_from_whitelist failed")?; + Ok(()) + }, + )?); - operations.push(measure_operation("load_whitelist", ops, |_| { - let whitelist = database.load_whitelist().context("load_whitelist failed")?; - drop(whitelist); - Ok(()) - })?); + operations.push(measure_operation( + "load_whitelist", + ops, + |_| Ok(()), + |()| { + let whitelist = database.load_whitelist().context("load_whitelist failed")?; + drop(whitelist); + Ok(()) + }, + )?); - operations.push(measure_operation("remove_info_hash_from_whitelist", ops, |index| { - let info_hash = info_hash_from_index(50_000 + index)?; - let _added_rows = database - .add_info_hash_to_whitelist(info_hash) - .context("failed to seed remove_info_hash_from_whitelist")?; - let _removed_rows = database - .remove_info_hash_from_whitelist(info_hash) - .context("remove_info_hash_from_whitelist failed")?; - Ok(()) - })?); + operations.push(measure_operation( + "remove_info_hash_from_whitelist", + ops, + |index| { + let info_hash = info_hash_from_index(50_000 + index)?; + let _added_rows = database + .add_info_hash_to_whitelist(info_hash) + .context("failed to seed remove_info_hash_from_whitelist")?; + Ok(info_hash) + }, + |info_hash| { + let _removed_rows = database + .remove_info_hash_from_whitelist(info_hash) + .context("remove_info_hash_from_whitelist failed")?; + Ok(()) + }, + )?); Ok(()) } diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs index 798c7ff8e..1f39eb853 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/sampling.rs @@ -8,22 +8,29 @@ use super::RawOperationSamples; /// Measures one database operation `ops` times and records elapsed samples. /// -/// The closure receives the iteration index so callers can generate distinct -/// fixture values when required. +/// Per-iteration fixture generation is performed by `setup` before timing +/// starts, so the recorded durations reflect only the database operation. /// /// # Errors /// -/// Returns an error if any operation invocation fails. -pub(super) fn measure_operation(name: impl Into, ops: usize, mut operation: F) -> Result +/// Returns an error if setup or any operation invocation fails. +pub(super) fn measure_operation( + name: impl Into, + ops: usize, + mut setup: S, + mut operation: F, +) -> Result where - F: FnMut(usize) -> Result<()>, + S: FnMut(usize) -> Result, + F: FnMut(T) -> Result<()>, { let name = name.into(); let mut samples = Vec::with_capacity(ops); for index in 0..ops { + let prepared = setup(index)?; let start = Instant::now(); - operation(index)?; + operation(prepared)?; samples.push(start.elapsed()); }