From b06ee0fa57b6a422476bf1f168e7e782dc435a8c Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 07:48:13 +0100 Subject: [PATCH 01/22] docs(issues): prefix 1525-08 spec with issue number --- docs/issues/1525-overhaul-persistence.md | 2 +- docs/issues/1721-1525-07-align-rust-and-db-types.md | 2 +- ...stgresql-driver.md => 1723-1525-08-add-postgresql-driver.md} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename docs/issues/{1525-08-add-postgresql-driver.md => 1723-1525-08-add-postgresql-driver.md} (100%) diff --git a/docs/issues/1525-overhaul-persistence.md b/docs/issues/1525-overhaul-persistence.md index 25fb2ec53..2f8c6340d 100644 --- a/docs/issues/1525-overhaul-persistence.md +++ b/docs/issues/1525-overhaul-persistence.md @@ -130,7 +130,7 @@ You can then browse or search it while working in the main repository. ### 8) Add PostgreSQL driver support -- Spec file: `docs/issues/1525-08-add-postgresql-driver.md` +- Spec file: `docs/issues/1723-1525-08-add-postgresql-driver.md` - Outcome: PostgreSQL support lands on top of the refactored and migration-backed persistence layer; PostgreSQL is added to the compatibility matrix (subissue 1) and qBittorrent E2E (subissue 2) test harnesses diff --git a/docs/issues/1721-1525-07-align-rust-and-db-types.md b/docs/issues/1721-1525-07-align-rust-and-db-types.md index 97614c104..6b03242b8 100644 --- a/docs/issues/1721-1525-07-align-rust-and-db-types.md +++ b/docs/issues/1721-1525-07-align-rust-and-db-types.md @@ -243,7 +243,7 @@ These tests extend the existing driver `#[cfg(test)]` modules. - EPIC: `#1525` - Subissue `1525-06`: `docs/issues/1719-1525-06-introduce-schema-migrations.md` — must be completed first (provides the migration framework) -- Subissue `1525-08`: `docs/issues/1525-08-add-postgresql-driver.md` — adds PostgreSQL +- Subissue `1525-08`: `docs/issues/1723-1525-08-add-postgresql-driver.md` — adds PostgreSQL migration files including the history-aligned no-op for this migration - Subissue `1525-03`: `docs/issues/1525-03-persistence-benchmarking.md` — benchmark baseline - Reference implementation branch: `josecelano:pr-1684-review` — see EPIC for checkout diff --git a/docs/issues/1525-08-add-postgresql-driver.md b/docs/issues/1723-1525-08-add-postgresql-driver.md similarity index 100% rename from docs/issues/1525-08-add-postgresql-driver.md rename to docs/issues/1723-1525-08-add-postgresql-driver.md From cd665bff71223241b0db2a98bf712a9282f247be Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 09:05:05 +0100 Subject: [PATCH 02/22] docs(issues): update 1723-1525-08 spec with user answers and implementation summary --- .../1723-1525-08-add-postgresql-driver.md | 338 +++++++++++++----- 1 file changed, 255 insertions(+), 83 deletions(-) diff --git a/docs/issues/1723-1525-08-add-postgresql-driver.md b/docs/issues/1723-1525-08-add-postgresql-driver.md index 71407fbbe..deafd990b 100644 --- a/docs/issues/1723-1525-08-add-postgresql-driver.md +++ b/docs/issues/1723-1525-08-add-postgresql-driver.md @@ -24,10 +24,15 @@ persistence layer is async (`1525-05`), schema-managed (`1525-06`), and correctl By the time this subissue is implemented: -- **1525-04** has split the monolithic `Database` trait into four narrow context traits - (`SchemaMigrator`, `TorrentMetricsStore`, `WhitelistStore`, `AuthKeyStore`) plus a blanket - `Database` aggregate supertrait. Both existing drivers (`Sqlite`, `Mysql`) satisfy `Database` - through the blanket impl. Consumers hold `Arc>`. +- **1525-04** and **1525-04b** together split the monolithic `Database` trait into four + narrow context traits (`SchemaMigrator`, `TorrentMetricsStore`, `WhitelistStore`, + `AuthKeyStore`) plus a blanket `Database` aggregate supertrait, and migrated all + production consumers to narrow traits. Both existing drivers (`Sqlite`, `Mysql`) satisfy + `Database` through the blanket impl. The factory (`initialize_database`) in + `databases/setup.rs` constructs the concrete driver once and returns a `DatabaseStores` + struct whose fields are `Arc` — production consumers never see + `Arc>`. The internal driver test helpers in `databases/driver/mod.rs` + still use `Arc>` as a convenience wrapper for the shared test suite. - **1525-05** has moved SQLite and MySQL to async `sqlx` connection pools. `r2d2`, `r2d2_sqlite`, `rusqlite`, and the `mysql` crate are gone. The `sqlx` dependency has `sqlite` and `mysql` @@ -342,25 +347,34 @@ PostgreSQL upsert syntax. There are no behavior differences relative to the othe In `packages/tracker-core/src/databases/driver/mod.rs`: -- Add `PostgreSQL` variant to the `Driver` enum. +- Add `PostgreSQL` variant to the `Driver` enum (and extend `as_str()` and `FromStr` to + recognize `"postgresql"`). - Add a `pub mod postgres;` declaration. -- Add a match arm in `build()`: - ```rust - Driver::PostgreSQL => { - let backend = Postgres::new(db_path)?; - Ok(Arc::new(Box::new(backend) as Box)) - } - ``` +There is no `build()` helper in this module. The concrete driver is constructed +directly in `setup.rs`. ### Database setup -In `packages/tracker-core/src/databases/setup.rs`, extend the configuration-to-internal -driver enum conversion: +In `packages/tracker-core/src/databases/setup.rs`: -```rust -torrust_tracker_configuration::Driver::PostgreSQL => Driver::PostgreSQL, -``` +- Extend the first `match` (config driver → internal `Driver` enum): + + ```rust + torrust_tracker_configuration::Driver::PostgreSQL => Driver::PostgreSQL, + ``` + +- Add a `Driver::PostgreSQL` arm to the second `match` (internal `Driver` → concrete + construction), mirroring the `Sqlite3` and `MySQL` arms: + + ```rust + Driver::PostgreSQL => { + use super::driver::postgres::Postgres; + let db = Arc::new(Postgres::new(&config.database.path).expect("Database driver build failed.")); + db.create_database_tables().await.expect("Could not create database tables."); + build_database_stores(db) + } + ``` ### Default configuration file @@ -380,16 +394,17 @@ All other sections remain the same as the existing container configs. Add an inline `#[cfg(test)]` module in `postgres.rs`. The test is guarded by an environment variable to avoid requiring a PostgreSQL container in every `cargo test` run. -**Environment variables**: +**Environment variables** (matching the MySQL driver pattern — testcontainers only): -| Variable | Purpose | Default | -| ------------------------------------------------ | ------------------------------------------ | ------------------------- | -| `TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST` | Enable the test (must be set to any value) | unset → test is skipped | -| `TORRUST_TRACKER_CORE_POSTGRES_DRIVER_URL` | Use an already-running PostgreSQL instance | unset → start a container | -| `TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE` | PostgreSQL Docker image name | `postgres` | -| `TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG` | PostgreSQL Docker image tag | `16` | +| Variable | Purpose | Default | +| ------------------------------------------------ | ------------------------------------------ | ----------------------- | +| `TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST` | Enable the test (must be set to any value) | unset → test is skipped | +| `TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG` | PostgreSQL Docker image tag | `16` | -**Test container defaults** (when no URL is provided): +No external-URL option. The test always starts a container, matching the MySQL driver +pattern. + +**Test container defaults**: ```text internal port: 5432 @@ -401,17 +416,26 @@ password: test Start the container using `testcontainers::GenericImage` (already a dev-dependency from MySQL tests). Set container env vars `POSTGRES_PASSWORD`, `POSTGRES_USER`, `POSTGRES_DB`. -**Test function skeleton**: +**Test function skeleton** (following the MySQL driver pattern): ```rust #[tokio::test] async fn run_postgres_driver_tests() -> Result<(), Box> { if std::env::var("TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST").is_err() { + println!("Skipping the PostgreSQL driver tests."); return Ok(()); } - let db_url = /* resolve from env or start container */; - let driver = Postgres::new(&db_url)?; - super::tests::run_tests(&driver).await; + + let postgres_configuration = PostgresConfiguration::default(); + let stopped_container = StoppedPostgresContainer::default(); + let container = stopped_container.run(&postgres_configuration).await.unwrap(); + + let host = container.get_host().await; + let port = container.get_host_port_ipv4().await; + let config = core_configuration(&host, port, &postgres_configuration); + + let driver = Arc::new(Box::new(Postgres::new(&config.database.path).unwrap()) as Box); + run_tests(&driver).await; Ok(()) } ``` @@ -490,10 +514,16 @@ Steps: - In `packages/tracker-core/src/databases/driver/mod.rs`: - Add `PostgreSQL` to the `Driver` enum. + - Extend `as_str()` to return `"postgresql"` for `PostgreSQL`. + - Extend `FromStr` to accept `"postgresql"` and update the error message to include it. - Add `pub mod postgres;`. - - Add the `Driver::PostgreSQL` arm in `build()`. - In `packages/tracker-core/src/databases/setup.rs`: - - Add `torrust_tracker_configuration::Driver::PostgreSQL => Driver::PostgreSQL`. + - Add `torrust_tracker_configuration::Driver::PostgreSQL => Driver::PostgreSQL` to the + first `match` (config → internal enum). + - Add the `Driver::PostgreSQL` arm to the second `match` (internal enum → concrete + construction), constructing `Arc::new(Postgres::new(...))` and calling + `create_database_tables()` then `build_database_stores(db)` — matching the existing + `Sqlite3` and `MySQL` arms exactly. Acceptance criteria: @@ -509,21 +539,20 @@ section above. Steps: - Implement `run_postgres_driver_tests` guarded by - `TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST`. -- Support both a pre-existing PostgreSQL instance (via - `TORRUST_TRACKER_CORE_POSTGRES_DRIVER_URL`) and a `testcontainers` container started - on demand. -- Default container tag: `16`. Image tag injection via + `TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST`, matching the MySQL driver test + structure exactly. +- Always start a `testcontainers::GenericImage` container (no external-URL fallback). +- Default container tag: `16`. Tag is overridable via `TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG` (enables the compatibility matrix loop in Task 6). - Call `tests::run_tests(&driver).await` — the shared test suite used by all backends. Acceptance criteria: -- [ ] `TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST` is unset → test returns immediately - without error. -- [ ] When the env var is set, the test starts a PostgreSQL container (or connects to the - provided URL), runs the shared test suite, and passes. +- [ ] `TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST` is unset → test prints skip message + and returns immediately without error. +- [ ] When the env var is set, the test starts a PostgreSQL container via testcontainers, + runs the shared test suite, and passes. - [ ] The container started by the test is removed unconditionally on completion or failure. ### Task 6 — Extend the compatibility matrix (completing subissue 1525-01) @@ -566,12 +595,14 @@ Acceptance criteria: included in the PR description. - [ ] The compatibility matrix exercises PostgreSQL 14, 15, 16, and 17 by default. -### Task 7 — Extend the qBittorrent E2E runner with PostgreSQL (completing subissue 1525-02) +### Task 7 — Extend the qBittorrent E2E runner with MySQL and PostgreSQL (completing subissue 1525-02) -The qBittorrent E2E runner introduced in subissue `1525-02` uses SQLite only. This task -extends it to support PostgreSQL and MySQL. MySQL E2E support (`--db-driver mysql`) is new -work introduced here — it was explicitly out of scope in `1525-02`. It is included here to -avoid a fourth subissue for a minor change and to keep all three backends consistent. +The qBittorrent E2E runner introduced in subissue `1525-02` uses SQLite only. The `Args` +struct in `src/console/ci/qbittorrent_e2e/runner.rs` has no `--db-driver` flag; +`config_builder.rs` defaults to an SQLite path for all runs. MySQL E2E support was +explicitly deferred in `1525-02` and has NOT been added since. This task adds +`--db-driver` support for all three backends: `sqlite3` (existing default, preserved), +`mysql` (new), and `postgresql` (new). Steps: @@ -648,34 +679,38 @@ Steps: `COPY --chmod=0555 ./share/container/entry_script_sh /usr/local/bin/entry.sh`; no `Containerfile` changes are needed. -- Update `compose.yaml` to support the PostgreSQL backend alongside the existing MySQL - service: - - Add a `postgres` service using `image: postgres:16`: - - ```yaml - postgres: - image: postgres:16 - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 3s - retries: 5 - start_period: 30s - environment: - - POSTGRES_PASSWORD=postgres - - POSTGRES_USER=postgres - - POSTGRES_DB=torrust_tracker - networks: - - server_side - volumes: - - postgres_data:/var/lib/postgresql/data - ``` +- Rename `compose.yaml` to `compose.mysql.yaml`. This file is used by + `.github/workflows/container.yaml` in the `docker compose build` step. Update the + workflow to pass `-f compose.mysql.yaml` so the rename is transparent to CI. + Update any documentation that references `compose.yaml` for the MySQL demo. + +- Add a new `compose.postgresql.yaml` for the PostgreSQL backend. Model it after the + renamed `compose.mysql.yaml` but replace the `mysql` service with a `postgres` service: + + ```yaml + postgres: + image: postgres:16 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 3s + retries: 5 + start_period: 30s + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=torrust_tracker + networks: + - server_side + volumes: + - postgres_data:/var/lib/postgresql/data + ``` - - Add `postgres` to the tracker service's `depends_on` list (alongside `mysql`) so the - tracker waits for whichever backend is healthy. Both DB services start; the tracker - connects to whichever backend the `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER` - env var selects. This is acceptable for a demo / developer compose file. + The tracker service in `compose.postgresql.yaml` should default to + `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=postgresql` and depend on + `postgres` only (not `mysql`). - - Add a `postgres_data` named volume to the `volumes:` section. +- Add a second `docker compose -f compose.postgresql.yaml build` step to the + `container.yaml` workflow so both compose files are validated in CI. - Update user-facing documentation to document PostgreSQL as a supported backend: - `README.md` — add `postgresql` to the list of supported database backends. @@ -693,11 +728,12 @@ Acceptance criteria: - [ ] `share/container/entry_script_sh` has a `postgresql` branch that selects `tracker.container.postgresql.toml`; the `else` error message lists all three supported backends. -- [ ] `compose.yaml` has a `postgres` service; the tracker service's `depends_on` includes - both `mysql` and `postgres`; a `postgres_data` volume is declared. -- [ ] `docker compose up` with - `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=postgresql` starts the tracker - successfully against the PostgreSQL container. +- [ ] `compose.yaml` is renamed to `compose.mysql.yaml`; `.github/workflows/container.yaml` + uses `-f compose.mysql.yaml`. +- [ ] `compose.postgresql.yaml` exists with a `postgres` service and a tracker service + that defaults to the PostgreSQL driver. +- [ ] `docker compose -f compose.postgresql.yaml up` starts the tracker successfully + against the PostgreSQL container. - [ ] The container configuration or its companion documentation (compose file or README) creates the `torrust_tracker` database (via `POSTGRES_DB` env var or equivalent) before the tracker is started. @@ -710,8 +746,10 @@ Acceptance criteria: ## Out of Scope -- Changing consumer wiring from `Arc>` to narrow trait objects. Deferred - until the MSRV reaches 1.76 (trait-object upcasting). +- Changing the internal driver test helpers (`databases/driver/mod.rs`) from + `Arc>` to narrow trait objects. Production consumers already use + narrow traits (`Arc`) via `DatabaseStores`; the test-helper wiring is + an internal concern and can be migrated separately. - PostgreSQL-specific performance tuning or connection pool size configuration beyond the default `PgPoolOptions` settings. - Down migrations (rollback support). @@ -742,16 +780,16 @@ Acceptance criteria: in tests, enabling the compatibility matrix loop. - [ ] `run-db-compatibility-matrix.sh` loops over `POSTGRES_VERSIONS` (default: `14 15 16 17`). -- [ ] The qBittorrent E2E runner completes a full download cycle with PostgreSQL. +- [ ] The qBittorrent E2E runner completes a full download cycle with both MySQL and + PostgreSQL (the `--db-driver` flag is added for all three backends). - [ ] The benchmark runner produces results for PostgreSQL; `docs/benchmarks/baseline.md` is updated. - [ ] `share/default/config/tracker.container.postgresql.toml` exists and is valid TOML. - [ ] `share/container/entry_script_sh` has a `postgresql` branch; the `else` error message lists all three supported backends. -- [ ] `compose.yaml` has a `postgres` service; the tracker service's `depends_on` includes - both `mysql` and `postgres`; `docker compose up` with - `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=postgresql` starts the tracker - successfully. +- [ ] `compose.yaml` is renamed to `compose.mysql.yaml`; `compose.postgresql.yaml` exists; + both are validated by `.github/workflows/container.yaml`; `docker compose -f + compose.postgresql.yaml up` starts the tracker successfully against PostgreSQL. - [ ] `project-words.txt` is up to date; `linter cspell` reports no failures. - [ ] `README.md` lists PostgreSQL as a supported database backend. - [ ] `docs/containers.md` documents how to run the tracker with PostgreSQL and states the @@ -762,6 +800,140 @@ Acceptance criteria: - [ ] `cargo machete` reports no unused dependencies. - [ ] `linter all` exits with code `0`. +## Implementation Questions + +The following questions must be answered before starting implementation. + +### Q1 — PR scope: single PR or phased? + +Do you want everything in this spec implemented in one PR, or split into phases +(e.g. core driver + migrations first, then QA/E2E/benchmark extensions)? + +**Answer**: + +I want one PR, but commits must be incremental and logically organized to allow for review in phases. +Each commit your be deployable (pass the pre-commit checks) and testable independently. + +### Q2 — CI scope for this subissue + +Should the PostgreSQL compatibility matrix be wired into +`.github/workflows/testing.yaml` now, or keep CI changes minimal and run +PostgreSQL checks manually for the first iteration? + +**Answer**: + +Yes, but that can be one of the independent tasks. + +### Q3 — MySQL support in the qBittorrent E2E runner + +The spec includes adding `--db-driver mysql` support to the qBittorrent E2E +runner as part of this subissue (Task 7). Should that stay coupled here, or +should this subissue deliver PostgreSQL-only E2E and defer MySQL E2E to a +follow-up? + +**Answer**: + +MySQL E2E was already added (confirmed). We have to add PostgreSQL to the E2E runner. +This can be an independent commit. Task 7 will add both `--db-driver` support and the +PostgreSQL E2E integration. + +### Q4 — Benchmark artifacts in this branch + +Should fresh benchmark results for PostgreSQL be generated and committed in +this same branch, or deferred until the driver is stable and a follow-up run +is done? + +**Answer**: + +Yes, after finishing the implementation and verifying the driver works, we can run benchmarks and update the baseline in the same branch. Again this can be another independent commit. + +### Q5 — `compose.yaml` database service strategy + +The spec says the tracker `depends_on` both `mysql` and `postgres` so both DB +services start regardless of which driver is selected. Alternatively, services +could be profile-based so only the selected backend starts. Which do you +prefer? + +**Answer**: + +Confirmed: the spec is correct. Rename `compose.yaml` → `compose.mysql.yaml`, add +`compose.postgresql.yaml` with the PostgreSQL service (tracker depends on `postgres` only), +and update `.github/workflows/container.yaml` to validate both files. This can be implemented +as part of Task 9 (containers and documentation updates). + +### Q6 — PostgreSQL driver test: testcontainers vs external URL + +The spec supports both a pre-existing PostgreSQL instance (via +`TORRUST_TRACKER_CORE_POSTGRES_DRIVER_URL`) and a testcontainers container. +Is this two-mode approach correct, or should the test always start a container +(matching the MySQL driver test pattern)? + +**Answer**: + +Match the MySQL driver test pattern: testcontainers only, no external-URL fallback. +This ensures consistent, isolated test environments across all three backends. + +### Q7 — Reference implementation alignment + +Should implementation prioritize parity with the reference branch +(`josecelano:pr-1684-review`) or prioritize the smallest clean diff against +the current refactored codebase, even where that diverges from the reference? + +**Answer**: + +Not at all. The reference implementation is a guide, not a spec. The implementation should prioritize the cleanest solution, even if that means diverging from the reference in some places. The reference may contain code that is no longer relevant or optimal in the context of the refactored codebase, and blindly following it could lead to unnecessary complexity or technical debt. By clean solutions, I mean solutions that are well-structured, maintainable, testable,and fit well with the existing codebase, even if they differ from the reference implementation. + +### Q8 — Implementation pace in this session + +After all answers are provided, should implementation proceed immediately and +run through lint/tests in the same session without pausing for interim review? + +**Answer**: + +No. Read replies, update spec, analyze code readiness, then begin implementation. +All commits must be incremental, deployable, and logically organized. + +--- + +## Implementation Summary + +Based on the answers above, the work will be delivered as **one PR with independent, +incremental commits** organized in the following phases: + +### Phase 1: Core driver (Tasks 1–6) + +These tasks establish the PostgreSQL driver fundamentals and must be completed first. +Each can be committed independently once it passes `linter all` and `cargo test`. + +- **Task 1**: Add `Driver::PostgreSQL` to configuration package +- **Task 2**: Add `Driver::PostgreSQL` variant to internal driver enum and `build()` factory +- **Task 3**: Implement `packages/tracker-core/src/databases/driver/postgres/mod.rs` (schema, + pools, traits) +- **Task 4**: Add migration files for PostgreSQL +- **Task 5**: Extend `packages/tracker-core/Cargo.toml` with `postgres` feature and + implement the driver tests +- **Task 6**: Extend the persistence benchmark runner (`BenchmarkResource::Postgres`) + +### Phase 2: Extended integration (Tasks 7–9) + +These tasks integrate PostgreSQL across the E2E harness, containers, and documentation. +Each can be a separate commit once Phase 1 is complete. + +- **Task 7**: Add `--db-driver` flag and PostgreSQL support to the qBittorrent E2E runner +- **Task 8**: Extend `.github/workflows/testing.yaml` with PostgreSQL compatibility matrix +- **Task 9**: Add container configs, update `entry_script_sh`, rename/add compose files, + update workflows and documentation + +### Phase 3: Verification (Task 10 — implicit) + +After all commits, run benchmarks and update baseline artifacts in a final commit. + +### Task dependencies + +**No hard blockers between phases.** Phase 1 tasks can run in parallel for code review +(all changes are scoped). Phase 2 tasks depend only on Phase 1 being complete. Benchmarks +(Phase 3) run last for data freshness. + ## References - EPIC: `#1525` — `docs/issues/1525-overhaul-persistence.md` From a0f9c001ff38433242924e8bb836f892386f4bfa Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 09:29:53 +0100 Subject: [PATCH 03/22] feat(tracker-core): add PostgreSQL database driver Implements a full PostgreSQL driver for the tracker, covering: - Driver::PostgreSQL variant in configuration and tracker-core - Four SQL migration files under migrations/postgresql/ - SchemaMigrator, AuthKeyStore, TorrentMetricsStore and WhitelistStore trait implementations using $1/$2 placeholders and ON CONFLICT UPSERT - Factory wiring in setup.rs - Benchmark runner support (postgres.rs + BenchmarkResource::Postgres) - Container entry script support for postgresql driver - Default container config tracker.container.postgresql.toml Closes #1723 Part of #1525 --- packages/configuration/src/v2_0_0/database.rs | 24 +- packages/tracker-core/Cargo.toml | 2 +- ...3000_torrust_tracker_create_all_tables.sql | 20 ++ ...rust_tracker_keys_valid_until_nullable.sql | 3 + ...er_new_torrent_aggregate_metrics_table.sql | 6 + ...orrust_tracker_widen_download_counters.sql | 5 + .../driver_bench/database/mod.rs | 7 +- .../driver_bench/database/postgres.rs | 87 ++++++ .../bin/persistence_benchmark/reporting.rs | 2 +- .../tracker-core/src/databases/driver/mod.rs | 7 +- .../driver/postgres/auth_key_store.rs | 125 ++++++++ .../src/databases/driver/postgres/mod.rs | 290 ++++++++++++++++++ .../driver/postgres/schema_migrator.rs | 35 +++ .../driver/postgres/torrent_metrics_store.rs | 106 +++++++ .../driver/postgres/whitelist_store.rs | 88 ++++++ packages/tracker-core/src/databases/setup.rs | 7 + share/container/entry_script_sh | 9 +- .../config/tracker.container.postgresql.toml | 32 ++ 18 files changed, 845 insertions(+), 10 deletions(-) create mode 100644 packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql create mode 100644 packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql create mode 100644 packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql create mode 100644 packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql create mode 100644 packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs create mode 100644 packages/tracker-core/src/databases/driver/postgres/auth_key_store.rs create mode 100644 packages/tracker-core/src/databases/driver/postgres/mod.rs create mode 100644 packages/tracker-core/src/databases/driver/postgres/schema_migrator.rs create mode 100644 packages/tracker-core/src/databases/driver/postgres/torrent_metrics_store.rs create mode 100644 packages/tracker-core/src/databases/driver/postgres/whitelist_store.rs create mode 100644 share/default/config/tracker.container.postgresql.toml diff --git a/packages/configuration/src/v2_0_0/database.rs b/packages/configuration/src/v2_0_0/database.rs index 457b3c925..ba34871e6 100644 --- a/packages/configuration/src/v2_0_0/database.rs +++ b/packages/configuration/src/v2_0_0/database.rs @@ -5,7 +5,7 @@ use url::Url; #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] pub struct Database { // Database configuration - /// Database driver. Possible values are: `sqlite3`, and `mysql`. + /// Database driver. Possible values are: `sqlite3`, `mysql`, and `postgresql`. #[serde(default = "Database::default_driver")] pub driver: Driver, @@ -14,6 +14,8 @@ pub struct Database { /// `./storage/tracker/lib/database/sqlite3.db`. /// For `mysql`, the format is `mysql://db_user:db_user_password@host:port/db_name`, for /// example: `mysql://root:password@localhost:3306/torrust`. + /// For `postgresql`, the format is `postgresql://db_user:db_user_password@host:port/db_name`, + /// for example: `postgresql://postgres:password@localhost:5432/torrust`. /// If the password contains reserved URL characters (for example `+` or `/`), /// percent-encode it in the URL. #[serde(default = "Database::default_path")] @@ -42,14 +44,14 @@ impl Database { /// /// # Panics /// - /// Will panic if the database path for `MySQL` is not a valid URL. + /// Will panic if the database path for `MySQL` or `PostgreSQL` is not a valid URL. pub fn mask_secrets(&mut self) { match self.driver { Driver::Sqlite3 => { // Nothing to mask } - Driver::MySQL => { - let mut url = Url::parse(&self.path).expect("path for MySQL driver should be a valid URL"); + Driver::MySQL | Driver::PostgreSQL => { + let mut url = Url::parse(&self.path).expect("path for MySQL/PostgreSQL driver should be a valid URL"); url.set_password(Some("***")).expect("url password should be changed"); self.path = url.to_string(); } @@ -65,6 +67,8 @@ pub enum Driver { Sqlite3, /// The `MySQL` database driver. MySQL, + /// The `PostgreSQL` database driver. + PostgreSQL, } #[cfg(test)] @@ -83,4 +87,16 @@ mod tests { assert_eq!(database.path, "mysql://root:***@localhost:3306/torrust".to_string()); } + + #[test] + fn it_should_allow_masking_the_postgresql_user_password() { + let mut database = Database { + driver: Driver::PostgreSQL, + path: "postgresql://postgres:password@localhost:5432/torrust".to_string(), + }; + + database.mask_secrets(); + + assert_eq!(database.path, "postgresql://postgres:***@localhost:5432/torrust".to_string()); + } } diff --git a/packages/tracker-core/Cargo.toml b/packages/tracker-core/Cargo.toml index e5d36e5fb..68b4f6bf4 100644 --- a/packages/tracker-core/Cargo.toml +++ b/packages/tracker-core/Cargo.toml @@ -29,7 +29,7 @@ mockall = "0" rand = "0" serde = { version = "1", features = [ "derive" ] } serde_json = { version = "1", features = [ "preserve_order" ] } -sqlx = { version = "0.8", features = [ "macros", "mysql", "runtime-tokio-native-tls", "sqlite" ] } +sqlx = { version = "0.8", features = [ "macros", "mysql", "postgres", "runtime-tokio-native-tls", "sqlite" ] } thiserror = "2" tokio = { version = "1", features = [ "macros", "net", "rt-multi-thread", "signal", "sync" ] } tokio-util = "0.7.15" diff --git a/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql b/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql new file mode 100644 index 000000000..509cb97ff --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql @@ -0,0 +1,20 @@ +CREATE TABLE + IF NOT EXISTS whitelist ( + id SERIAL PRIMARY KEY, + info_hash VARCHAR(40) NOT NULL UNIQUE + ); + +-- todo: rename to `torrent_metrics` +CREATE TABLE + IF NOT EXISTS torrents ( + id SERIAL PRIMARY KEY, + info_hash VARCHAR(40) NOT NULL UNIQUE, + completed INTEGER DEFAULT 0 NOT NULL + ); + +CREATE TABLE + IF NOT EXISTS keys ( + id SERIAL PRIMARY KEY, + key VARCHAR(32) NOT NULL UNIQUE, + valid_until INTEGER NOT NULL + ); \ No newline at end of file diff --git a/packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql b/packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql new file mode 100644 index 000000000..54080a0af --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20240730183500_torrust_tracker_keys_valid_until_nullable.sql @@ -0,0 +1,3 @@ +ALTER TABLE keys +ALTER COLUMN valid_until +DROP NOT NULL; \ No newline at end of file diff --git a/packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql b/packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql new file mode 100644 index 000000000..28c69becd --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20250527093000_torrust_tracker_new_torrent_aggregate_metrics_table.sql @@ -0,0 +1,6 @@ +CREATE TABLE + IF NOT EXISTS torrent_aggregate_metrics ( + id SERIAL PRIMARY KEY, + metric_name VARCHAR(50) NOT NULL UNIQUE, + value INTEGER DEFAULT 0 NOT NULL + ); \ No newline at end of file diff --git a/packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql b/packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql new file mode 100644 index 000000000..7ca1e4aa1 --- /dev/null +++ b/packages/tracker-core/migrations/postgresql/20260409120000_torrust_tracker_widen_download_counters.sql @@ -0,0 +1,5 @@ +ALTER TABLE torrents +ALTER COLUMN completed TYPE BIGINT; + +ALTER TABLE torrent_aggregate_metrics +ALTER COLUMN value TYPE BIGINT; \ No newline at end of file diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs index 083d735a4..582f68d21 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/mod.rs @@ -8,6 +8,7 @@ use bittorrent_tracker_core::databases::SchemaMigrator; use testcontainers::{ContainerAsync, GenericImage}; mod mysql; +mod postgres; mod sqlite; pub(super) struct ActiveDatabase { @@ -18,6 +19,7 @@ pub(super) struct ActiveDatabase { enum BenchmarkResource { Sqlite(PathBuf), Mysql(Box>), + Postgres(Box>), } impl ActiveDatabase { @@ -29,12 +31,13 @@ impl ActiveDatabase { /// /// # Errors /// - /// Returns an error if the `MySQL` container cannot be started or queried for + /// Returns an error if the `MySQL` or `PostgreSQL` container cannot be started or queried for /// connection details. pub(super) async fn new(driver: Driver, db_version: &str) -> Result { match driver { Driver::Sqlite3 => Ok(sqlite::initialize().await), Driver::MySQL => mysql::initialize(db_version).await, + Driver::PostgreSQL => postgres::initialize(db_version).await, } } } @@ -48,7 +51,7 @@ impl Drop for ActiveDatabase { Some(BenchmarkResource::Sqlite(path)) => { let _removed_file_result = std::fs::remove_file(path); } - Some(BenchmarkResource::Mysql(container)) => { + Some(BenchmarkResource::Mysql(container) | BenchmarkResource::Postgres(container)) => { drop(container); } None => {} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs new file mode 100644 index 000000000..62d79df33 --- /dev/null +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs @@ -0,0 +1,87 @@ +use std::str::FromStr; +use std::time::Duration; + +use anyhow::{Context, Result}; +use bittorrent_tracker_core::databases::setup::initialize_database; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use testcontainers::core::wait::LogWaitStrategy; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; +use torrust_tracker_configuration as configuration; + +use super::{ActiveDatabase, BenchmarkResource}; + +/// Maximum number of connect-and-ping attempts after the container is reported +/// ready. +const READINESS_PING_RETRIES: usize = 30; +/// Delay between readiness-ping attempts. +const READINESS_PING_INTERVAL: Duration = Duration::from_millis(500); + +pub(super) async fn initialize(db_version: &str) -> Result { + let postgres_container = GenericImage::new("postgres", db_version) + .with_exposed_port(5432.tcp()) + .with_wait_for(WaitFor::Log( + LogWaitStrategy::stderr("database system is ready to accept connections").with_times(2), + )) + .with_env_var("POSTGRES_PASSWORD", "test") + .with_env_var("POSTGRES_DB", "torrust_tracker_bench") + .with_env_var("POSTGRES_USER", "root") + .start() + .await + .context("failed to start postgres test container")?; + + let host = postgres_container + .get_host() + .await + .context("failed to resolve postgres container host")?; + let port = postgres_container + .get_host_port_ipv4(5432) + .await + .context("failed to resolve postgres container host port")?; + + let postgres_database_url = format!("postgresql://root:test@{host}:{port}/torrust_tracker_bench"); + + wait_until_postgres_accepts_connections(&postgres_database_url) + .await + .context("postgres container did not accept connections in time")?; + + let mut config = configuration::Core::default(); + config.database.driver = configuration::Driver::PostgreSQL; + config.database.path = postgres_database_url; + let database = initialize_database(&config).await; + + Ok(ActiveDatabase { + database: Some(database), + resource: Some(BenchmarkResource::Postgres(Box::new(postgres_container))), + }) +} + +async fn wait_until_postgres_accepts_connections(database_url: &str) -> Result<()> { + let options = PgConnectOptions::from_str(database_url).context("invalid postgres benchmark URL")?; + + let mut last_error: Option = None; + + for _ in 0..READINESS_PING_RETRIES { + match PgPoolOptions::new().max_connections(1).connect_with(options.clone()).await { + Ok(pool) => { + if let Err(error) = sqlx::query("SELECT 1").execute(&pool).await { + last_error = Some(error); + } else { + pool.close().await; + return Ok(()); + } + } + Err(error) => { + last_error = Some(error); + } + } + + tokio::time::sleep(READINESS_PING_INTERVAL).await; + } + + Err(anyhow::anyhow!( + "postgres still not accepting connections after {READINESS_PING_RETRIES} attempts; last error: {error}", + error = last_error.map_or_else(|| "".to_string(), |e| e.to_string()) + )) +} diff --git a/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs b/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs index 10ea7ddb1..e664d51f0 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs @@ -17,7 +17,7 @@ pub fn build_report( ) -> report::BenchReport { let normalized_db_version = match driver { Driver::Sqlite3 => "-".to_string(), - Driver::MySQL => db_version.to_string(), + Driver::MySQL | Driver::PostgreSQL => db_version.to_string(), }; let meta = report::ReportMeta::from_run_context(driver.as_str(), &normalized_db_version, ops, timings_ms); diff --git a/packages/tracker-core/src/databases/driver/mod.rs b/packages/tracker-core/src/databases/driver/mod.rs index 147275f30..bc1fa7926 100644 --- a/packages/tracker-core/src/databases/driver/mod.rs +++ b/packages/tracker-core/src/databases/driver/mod.rs @@ -22,6 +22,8 @@ pub enum Driver { Sqlite3, /// The `MySQL` database driver. MySQL, + /// The `PostgreSQL` database driver. + PostgreSQL, } impl Driver { @@ -31,6 +33,7 @@ impl Driver { match self { Self::Sqlite3 => "sqlite3", Self::MySQL => "mysql", + Self::PostgreSQL => "postgresql", } } } @@ -42,12 +45,14 @@ impl FromStr for Driver { match value { "sqlite3" => Ok(Self::Sqlite3), "mysql" => Ok(Self::MySQL), - _ => Err("driver must be one of: sqlite3, mysql".to_string()), + "postgresql" => Ok(Self::PostgreSQL), + _ => Err("driver must be one of: sqlite3, mysql, postgresql".to_string()), } } } pub mod mysql; +pub mod postgres; pub mod sqlite; #[cfg(test)] diff --git a/packages/tracker-core/src/databases/driver/postgres/auth_key_store.rs b/packages/tracker-core/src/databases/driver/postgres/auth_key_store.rs new file mode 100644 index 000000000..604ac4608 --- /dev/null +++ b/packages/tracker-core/src/databases/driver/postgres/auth_key_store.rs @@ -0,0 +1,125 @@ +use ::sqlx::Row; +use async_trait::async_trait; +use torrust_tracker_primitives::DurationSinceUnixEpoch; + +use super::{Postgres, DRIVER}; +use crate::authentication::{self, Key}; +use crate::databases::error::Error; +use crate::databases::AuthKeyStore; + +#[async_trait] +impl AuthKeyStore for Postgres { + async fn load_keys(&self) -> Result, Error> { + let rows = ::sqlx::query("SELECT key, valid_until FROM keys") + .fetch_all(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + rows.into_iter() + .map(|row| { + let key_value: String = row.try_get("key").map_err(|e| (e, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|e| (e, DRIVER))?; + + let parsed_key = key_value.parse::().map_err(|e| Error::MalformedDatabaseRecord { + message: e.to_string(), + driver: DRIVER, + })?; + + Ok(authentication::PeerKey { + key: parsed_key, + valid_until: valid_until.map(parse_valid_until).transpose()?, + }) + }) + .collect() + } + + async fn get_key_from_keys(&self, key: &Key) -> Result, Error> { + let maybe_row = ::sqlx::query("SELECT key, valid_until FROM keys WHERE key = $1") + .bind(key.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + maybe_row + .map(|row| { + let key_value: String = row.try_get("key").map_err(|e| (e, DRIVER))?; + let valid_until: Option = row.try_get("valid_until").map_err(|e| (e, DRIVER))?; + + let parsed_key = key_value.parse::().map_err(|e| Error::MalformedDatabaseRecord { + message: e.to_string(), + driver: DRIVER, + })?; + + Ok(authentication::PeerKey { + key: parsed_key, + valid_until: valid_until.map(parse_valid_until).transpose()?, + }) + }) + .transpose() + } + + async fn add_key_to_keys(&self, auth_key: &authentication::PeerKey) -> Result { + let valid_until = auth_key + .valid_until + .map(|value| { + i64::try_from(value.as_secs()).map_err(|e| Error::MalformedDatabaseRecord { + message: e.to_string(), + driver: DRIVER, + }) + }) + .transpose()?; + + let insert = ::sqlx::query("INSERT INTO keys (key, valid_until) VALUES ($1, $2)") + .bind(auth_key.key.to_string()) + .bind(valid_until) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))? + .rows_affected(); + + if insert == 0 { + Err(Error::InsertFailed { + location: std::panic::Location::caller(), + driver: DRIVER, + }) + } else { + usize::try_from(insert).map_err(|e| Error::MalformedDatabaseRecord { + message: format!("rows_affected does not fit in usize: {e}"), + driver: DRIVER, + }) + } + } + + async fn remove_key_from_keys(&self, key: &Key) -> Result { + let deleted = ::sqlx::query("DELETE FROM keys WHERE key = $1") + .bind(key.to_string()) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))? + .rows_affected(); + + if deleted == 1 { + Ok(1) + } else { + Err(Error::DeleteFailed { + location: std::panic::Location::caller(), + error_code: usize::try_from(deleted).unwrap_or(0), + driver: DRIVER, + }) + } + } +} + +/// Convert a signed seconds value loaded from the database into a +/// [`DurationSinceUnixEpoch`]. +/// +/// Negative values indicate a corrupted record (timestamps before the Unix +/// epoch are not representable) and are rejected as +/// [`Error::MalformedDatabaseRecord`]. +fn parse_valid_until(value: i64) -> Result { + let secs = u64::try_from(value).map_err(|_| Error::MalformedDatabaseRecord { + message: format!("negative valid_until timestamp: {value}"), + driver: DRIVER, + })?; + Ok(DurationSinceUnixEpoch::from_secs(secs)) +} diff --git a/packages/tracker-core/src/databases/driver/postgres/mod.rs b/packages/tracker-core/src/databases/driver/postgres/mod.rs new file mode 100644 index 000000000..c51ff2ddc --- /dev/null +++ b/packages/tracker-core/src/databases/driver/postgres/mod.rs @@ -0,0 +1,290 @@ +//! The `PostgreSQL` database driver. +use std::str::FromStr; + +use ::sqlx::migrate::Migrator; +use ::sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use ::sqlx::{PgPool, Row}; +use torrust_tracker_primitives::NumberOfDownloads; + +use super::{Driver, Error}; + +mod auth_key_store; +mod schema_migrator; +mod torrent_metrics_store; +mod whitelist_store; + +const DRIVER: Driver = Driver::PostgreSQL; + +/// Embedded `sqlx` migrator for the `PostgreSQL` backend. +/// +/// All `.sql` files under `migrations/postgresql/` are compiled into the binary at +/// build time and applied in timestamp order by `MIGRATOR.run(&pool)`. +pub(super) static MIGRATOR: Migrator = ::sqlx::migrate!("migrations/postgresql"); + +/// `PostgreSQL` driver implementation. +/// +/// This struct encapsulates an async `sqlx` connection pool for `PostgreSQL`. +/// It implements the [`Database`] trait to provide persistence operations. +pub(crate) struct Postgres { + pool: PgPool, +} + +impl Postgres { + pub fn new(db_path: &str) -> Result { + let options = PgConnectOptions::from_str(db_path).map_err(|e| (e, DRIVER))?; + + let pool = PgPoolOptions::new().connect_lazy_with(options); + + Ok(Self { pool }) + } + + async fn load_torrent_aggregate_metric(&self, metric_name: &str) -> Result, Error> { + let maybe_row = ::sqlx::query("SELECT value FROM torrent_aggregate_metrics WHERE metric_name = $1") + .bind(metric_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + maybe_row + .map(|row| { + let value: i64 = row.try_get("value").map_err(|e| (e, DRIVER))?; + u32::try_from(value).map_err(|e| Error::MalformedDatabaseRecord { + message: e.to_string(), + driver: DRIVER, + }) + }) + .transpose() + } + + async fn save_torrent_aggregate_metric(&self, metric_name: &str, completed: NumberOfDownloads) -> Result<(), Error> { + // `ON CONFLICT ... DO UPDATE SET` may legitimately report `rows_affected() == 0` + // when the row already exists with the same value (no-op update), so we + // do not treat 0 as a failure here. A real failure surfaces as `Err` + // from `execute()`. + ::sqlx::query( + "INSERT INTO torrent_aggregate_metrics (metric_name, value) VALUES ($1, $2) \ + ON CONFLICT (metric_name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(metric_name) + .bind(i64::from(completed)) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + Ok(()) + } +} + +#[cfg(all(test, feature = "db-compatibility-tests"))] +mod tests { + use std::sync::Arc; + + use testcontainers::core::IntoContainerPort; + use testcontainers::runners::AsyncRunner; + use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + use torrust_tracker_configuration::Core; + + use super::Postgres; + use crate::databases::driver::tests::run_tests; + use crate::databases::traits::Database; + use crate::test_helpers::tests::random_info_hash; + + #[derive(Debug, Default)] + struct StoppedPostgresContainer {} + + impl StoppedPostgresContainer { + async fn run( + self, + config: &PostgresConfiguration, + ) -> Result> { + let image_tag = std::env::var("TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG").unwrap_or_else(|_| "16".to_string()); + + let container = GenericImage::new("postgres", image_tag.as_str()) + .with_exposed_port(config.internal_port.tcp()) + .with_env_var("POSTGRES_PASSWORD", config.db_password.clone()) + .with_env_var("POSTGRES_USER", config.db_user.clone()) + .with_env_var("POSTGRES_DB", config.database.clone()) + .start() + .await?; + + Ok(RunningPostgresContainer::new(container, config.internal_port)) + } + } + + struct RunningPostgresContainer { + container: ContainerAsync, + internal_port: u16, + } + + impl RunningPostgresContainer { + fn new(container: ContainerAsync, internal_port: u16) -> Self { + Self { + container, + internal_port, + } + } + + async fn stop(self) { + self.container.stop().await.unwrap(); + } + + async fn get_host(&self) -> url::Host { + self.container.get_host().await.unwrap() + } + + async fn get_host_port_ipv4(&self) -> u16 { + self.container.get_host_port_ipv4(self.internal_port).await.unwrap() + } + } + + impl Default for PostgresConfiguration { + fn default() -> Self { + Self { + internal_port: 5432, + database: "torrust_tracker_test".to_string(), + db_user: "postgres".to_string(), + db_password: "test".to_string(), + } + } + } + + struct PostgresConfiguration { + pub internal_port: u16, + pub database: String, + pub db_user: String, + pub db_password: String, + } + + fn core_configuration(host: &url::Host, port: u16, postgres_configuration: &PostgresConfiguration) -> Core { + let mut config = Core::default(); + + let database = postgres_configuration.database.clone(); + let db_user = postgres_configuration.db_user.clone(); + let db_password = postgres_configuration.db_password.clone(); + + config.database.path = format!("postgres://{db_user}:{db_password}@{host}:{port}/{database}"); + + config + } + + fn initialize_driver(config: &Core) -> Arc> { + Arc::new(Box::new(Postgres::new(&config.database.path).unwrap())) + } + + // This test is invoked by `.github/workflows/testing.yaml` in the + // `database-compatibility` job to validate supported PostgreSQL versions. + #[tokio::test] + async fn run_postgres_driver_tests() -> Result<(), Box> { + if std::env::var("TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST").is_err() { + println!("Skipping the PostgreSQL driver tests."); + return Ok(()); + } + + let postgres_configuration = PostgresConfiguration::default(); + + let stopped_postgres_container = StoppedPostgresContainer::default(); + + let postgres_container = stopped_postgres_container.run(&postgres_configuration).await.unwrap(); + + let host = postgres_container.get_host().await; + let port = postgres_container.get_host_port_ipv4().await; + + let config = core_configuration(&host, port, &postgres_configuration); + + let driver = initialize_driver(&config); + + run_tests(&driver).await; + + // Idempotency: a second `create_database_tables()` call must be a + // no-op (embedded `sqlx` migrator skips migrations already recorded + // in `_sqlx_migrations`). + driver + .create_database_tables() + .await + .expect("second migration run should be a no-op"); + + // PostgreSQL has no legacy pre-v4 databases, so we skip the + // legacy bootstrap test. PostgreSQL support was added in v4+. + driver.drop_database_tables().await.expect("drop tables for fresh test"); + + let raw_pool = ::sqlx::postgres::PgPoolOptions::new() + .connect(&config.database.path) + .await + .expect("connect to postgres for raw DDL"); + create_legacy_pre_v4_schema(&raw_pool).await; + + driver + .create_database_tables() + .await + .expect("fresh schema creation should succeed"); + + let recorded: i64 = ::sqlx::query_scalar("SELECT COUNT(*) FROM _sqlx_migrations") + .fetch_one(&raw_pool) + .await + .expect("count _sqlx_migrations"); + assert_eq!(recorded, 4, "all migrations should be recorded after migrator run"); + + assert_postgres_column_type(&raw_pool, "torrents", "completed", "bigint").await; + assert_postgres_column_type(&raw_pool, "torrent_aggregate_metrics", "value", "bigint").await; + + let above_i32_max = 2_200_000_000_u32; + let info_hash = random_info_hash(); + + driver + .save_torrent_downloads(&info_hash, above_i32_max) + .await + .expect("save torrent downloads above i32::MAX should succeed"); + let loaded_torrent_downloads = driver + .load_torrent_downloads(&info_hash) + .await + .expect("load torrent downloads above i32::MAX should succeed"); + assert_eq!(loaded_torrent_downloads, Some(above_i32_max)); + + driver + .save_global_downloads(above_i32_max) + .await + .expect("save global downloads above i32::MAX should succeed"); + let loaded_global_downloads = driver + .load_global_downloads() + .await + .expect("load global downloads above i32::MAX should succeed"); + assert_eq!(loaded_global_downloads, Some(above_i32_max)); + + drop(raw_pool); + + postgres_container.stop().await; + + Ok(()) + } + + /// Create a minimal schema for `PostgreSQL`. + /// + /// `PostgreSQL` support was added in v4, so there are no pre-v4 databases. + /// This helper creates a fresh schema to test idempotency of the migrator. + async fn create_legacy_pre_v4_schema(pool: &::sqlx::PgPool) { + for stmt in [ + "CREATE TABLE IF NOT EXISTS whitelist (id SERIAL PRIMARY KEY, info_hash VARCHAR(40) NOT NULL UNIQUE)", + "CREATE TABLE IF NOT EXISTS torrents (id SERIAL PRIMARY KEY, info_hash VARCHAR(40) NOT NULL UNIQUE, completed INTEGER DEFAULT 0 NOT NULL)", + "CREATE TABLE IF NOT EXISTS keys (id SERIAL PRIMARY KEY, key VARCHAR(32) NOT NULL UNIQUE, valid_until INTEGER NOT NULL)", + "CREATE TABLE IF NOT EXISTS torrent_aggregate_metrics (id SERIAL PRIMARY KEY, metric_name VARCHAR(50) NOT NULL UNIQUE, value INTEGER DEFAULT 0 NOT NULL)", + ] { + ::sqlx::query(stmt).execute(pool).await.expect("schema DDL"); + } + } + + async fn assert_postgres_column_type(pool: &::sqlx::PgPool, table: &str, column: &str, expected_type: &str) { + let data_type: String = + ::sqlx::query_scalar("SELECT data_type FROM information_schema.columns WHERE table_name = $1 AND column_name = $2") + .bind(table) + .bind(column) + .fetch_one(pool) + .await + .expect("column type query should succeed"); + + assert_eq!( + data_type.to_lowercase(), + expected_type, + "{table}.{column} should be {expected_type}" + ); + } +} diff --git a/packages/tracker-core/src/databases/driver/postgres/schema_migrator.rs b/packages/tracker-core/src/databases/driver/postgres/schema_migrator.rs new file mode 100644 index 000000000..8c2bd0393 --- /dev/null +++ b/packages/tracker-core/src/databases/driver/postgres/schema_migrator.rs @@ -0,0 +1,35 @@ +use async_trait::async_trait; + +use super::{Postgres, DRIVER, MIGRATOR}; +use crate::databases::error::Error; +use crate::databases::SchemaMigrator; + +#[async_trait] +impl SchemaMigrator for Postgres { + async fn create_database_tables(&self) -> Result<(), Error> { + // `PostgreSQL` has no pre-v4 databases, so we skip legacy bootstrap + // and run the embedded migrator directly. + MIGRATOR.run(&self.pool).await.map_err(|e| (e, DRIVER))?; + Ok(()) + } + + async fn drop_database_tables(&self) -> Result<(), Error> { + // `IF EXISTS` keeps test teardown safe across partial schemas. + // `_sqlx_migrations` is created by the embedded `sqlx` migrator and + // must be dropped here so the next `create_database_tables()` call + // re-applies every migration from a clean state. + let statements = [ + "DROP TABLE IF EXISTS _sqlx_migrations;", + "DROP TABLE IF EXISTS torrent_aggregate_metrics;", + "DROP TABLE IF EXISTS whitelist;", + "DROP TABLE IF EXISTS torrents;", + "DROP TABLE IF EXISTS keys;", + ]; + + for stmt in statements { + ::sqlx::query(stmt).execute(&self.pool).await.map_err(|e| (e, DRIVER))?; + } + + Ok(()) + } +} diff --git a/packages/tracker-core/src/databases/driver/postgres/torrent_metrics_store.rs b/packages/tracker-core/src/databases/driver/postgres/torrent_metrics_store.rs new file mode 100644 index 000000000..d96fd2268 --- /dev/null +++ b/packages/tracker-core/src/databases/driver/postgres/torrent_metrics_store.rs @@ -0,0 +1,106 @@ +use std::str::FromStr; + +use ::sqlx::Row; +use async_trait::async_trait; +use bittorrent_primitives::info_hash::InfoHash; +use torrust_tracker_primitives::{NumberOfDownloads, NumberOfDownloadsBTreeMap}; + +use super::{Postgres, DRIVER}; +use crate::databases::driver::TORRENTS_DOWNLOADS_TOTAL; +use crate::databases::error::Error; +use crate::databases::TorrentMetricsStore; + +#[async_trait] +impl TorrentMetricsStore for Postgres { + async fn load_all_torrents_downloads(&self) -> Result { + let rows = ::sqlx::query("SELECT info_hash, completed FROM torrents") + .fetch_all(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + rows.into_iter() + .map(|row| { + let info_hash_value: String = row.try_get("info_hash").map_err(|e| (e, DRIVER))?; + let completed: i64 = row.try_get("completed").map_err(|e| (e, DRIVER))?; + let completed = u32::try_from(completed).map_err(|e| Error::MalformedDatabaseRecord { + message: e.to_string(), + driver: DRIVER, + })?; + + InfoHash::from_str(&info_hash_value) + .map(|info_hash| (info_hash, completed)) + .map_err(|e| Error::MalformedDatabaseRecord { + message: format!("{e:?}"), + driver: DRIVER, + }) + }) + .collect::, Error>>() + .map(|v| v.iter().copied().collect()) + } + + async fn load_torrent_downloads(&self, info_hash: &InfoHash) -> Result, Error> { + let maybe_row = ::sqlx::query("SELECT completed FROM torrents WHERE info_hash = $1") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + maybe_row + .map(|row| { + let completed: i64 = row.try_get("completed").map_err(|e| (e, DRIVER))?; + u32::try_from(completed).map_err(|e| Error::MalformedDatabaseRecord { + message: e.to_string(), + driver: DRIVER, + }) + }) + .transpose() + } + + async fn save_torrent_downloads(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error> { + // `ON CONFLICT ... DO UPDATE SET` may legitimately report `rows_affected() == 0` + // when the row already exists with the same value (no-op update), so we + // do not treat 0 as a failure here. A real failure surfaces as `Err` + // from `execute()`. + ::sqlx::query( + "INSERT INTO torrents (info_hash, completed) VALUES ($1, $2) \ + ON CONFLICT (info_hash) DO UPDATE SET completed = EXCLUDED.completed", + ) + .bind(info_hash.to_string()) + .bind(i64::from(completed)) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + Ok(()) + } + + async fn increase_downloads_for_torrent(&self, info_hash: &InfoHash) -> Result<(), Error> { + ::sqlx::query("UPDATE torrents SET completed = completed + 1 WHERE info_hash = $1") + .bind(info_hash.to_string()) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + Ok(()) + } + + async fn load_global_downloads(&self) -> Result, Error> { + self.load_torrent_aggregate_metric(TORRENTS_DOWNLOADS_TOTAL).await + } + + async fn save_global_downloads(&self, downloaded: NumberOfDownloads) -> Result<(), Error> { + self.save_torrent_aggregate_metric(TORRENTS_DOWNLOADS_TOTAL, downloaded).await + } + + async fn increase_global_downloads(&self) -> Result<(), Error> { + let metric_name = TORRENTS_DOWNLOADS_TOTAL; + + ::sqlx::query("UPDATE torrent_aggregate_metrics SET value = value + 1 WHERE metric_name = $1") + .bind(metric_name) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + Ok(()) + } +} diff --git a/packages/tracker-core/src/databases/driver/postgres/whitelist_store.rs b/packages/tracker-core/src/databases/driver/postgres/whitelist_store.rs new file mode 100644 index 000000000..a8d42475f --- /dev/null +++ b/packages/tracker-core/src/databases/driver/postgres/whitelist_store.rs @@ -0,0 +1,88 @@ +use std::panic::Location; +use std::str::FromStr; + +use ::sqlx::Row; +use async_trait::async_trait; +use bittorrent_primitives::info_hash::InfoHash; + +use super::{Postgres, DRIVER}; +use crate::databases::error::Error; +use crate::databases::WhitelistStore; + +#[async_trait] +impl WhitelistStore for Postgres { + async fn load_whitelist(&self) -> Result, Error> { + let rows = ::sqlx::query("SELECT info_hash FROM whitelist") + .fetch_all(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + rows.into_iter() + .map(|row| { + let value: String = row.try_get("info_hash").map_err(|e| (e, DRIVER))?; + InfoHash::from_str(&value).map_err(|e| Error::MalformedDatabaseRecord { + message: format!("{e:?}"), + driver: DRIVER, + }) + }) + .collect() + } + + async fn get_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result, Error> { + let maybe_row = ::sqlx::query("SELECT info_hash FROM whitelist WHERE info_hash = $1") + .bind(info_hash.to_hex_string()) + .fetch_optional(&self.pool) + .await + .map_err(|e| (e, DRIVER))?; + + maybe_row + .map(|row| { + let value: String = row.try_get("info_hash").map_err(|e| (e, DRIVER))?; + InfoHash::from_str(&value).map_err(|e| Error::MalformedDatabaseRecord { + message: format!("{e:?}"), + driver: DRIVER, + }) + }) + .transpose() + } + + async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { + let insert = ::sqlx::query("INSERT INTO whitelist (info_hash) VALUES ($1)") + .bind(info_hash.to_string()) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))? + .rows_affected(); + + if insert == 0 { + Err(Error::InsertFailed { + location: Location::caller(), + driver: DRIVER, + }) + } else { + usize::try_from(insert).map_err(|e| Error::MalformedDatabaseRecord { + message: format!("rows_affected does not fit in usize: {e}"), + driver: DRIVER, + }) + } + } + + async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { + let deleted = ::sqlx::query("DELETE FROM whitelist WHERE info_hash = $1") + .bind(info_hash.to_string()) + .execute(&self.pool) + .await + .map_err(|e| (e, DRIVER))? + .rows_affected(); + + if deleted == 1 { + Ok(1) + } else { + Err(Error::DeleteFailed { + location: Location::caller(), + error_code: usize::try_from(deleted).unwrap_or(0), + driver: DRIVER, + }) + } + } +} diff --git a/packages/tracker-core/src/databases/setup.rs b/packages/tracker-core/src/databases/setup.rs index c09a754e3..8c94c1586 100644 --- a/packages/tracker-core/src/databases/setup.rs +++ b/packages/tracker-core/src/databases/setup.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use torrust_tracker_configuration::Core; use super::driver::mysql::Mysql; +use super::driver::postgres::Postgres; use super::driver::sqlite::Sqlite; use super::driver::Driver; use super::traits::{AuthKeyStore, SchemaMigrator, TorrentMetricsStore, WhitelistStore}; @@ -91,6 +92,7 @@ pub async fn initialize_database(config: &Core) -> DatabaseStores { let driver = match config.database.driver { torrust_tracker_configuration::Driver::Sqlite3 => Driver::Sqlite3, torrust_tracker_configuration::Driver::MySQL => Driver::MySQL, + torrust_tracker_configuration::Driver::PostgreSQL => Driver::PostgreSQL, }; match driver { @@ -104,6 +106,11 @@ pub async fn initialize_database(config: &Core) -> DatabaseStores { db.create_database_tables().await.expect("Could not create database tables."); build_database_stores(db) } + Driver::PostgreSQL => { + let db = Arc::new(Postgres::new(&config.database.path).expect("Database driver build failed.")); + db.create_database_tables().await.expect("Could not create database tables."); + build_database_stores(db) + } } } diff --git a/share/container/entry_script_sh b/share/container/entry_script_sh index 32cdfe33d..eb4ebce14 100644 --- a/share/container/entry_script_sh +++ b/share/container/entry_script_sh @@ -42,9 +42,16 @@ if [ -n "$TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER" ]; then # Select default MySQL configuration default_config="/usr/share/torrust/default/config/tracker.container.mysql.toml" + elif cmp_lc "$TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER" "postgresql"; then + + # (no database file needed for PostgreSQL) + + # Select default PostgreSQL configuration + default_config="/usr/share/torrust/default/config/tracker.container.postgresql.toml" + else echo "Error: Unsupported Database Type: \"$TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER\"." - echo "Please Note: Supported Database Types: \"sqlite3\", \"mysql\"." + echo "Please Note: Supported Database Types: \"sqlite3\", \"mysql\", \"postgresql\"." exit 1 fi else diff --git a/share/default/config/tracker.container.postgresql.toml b/share/default/config/tracker.container.postgresql.toml new file mode 100644 index 000000000..ec3a9bdbe --- /dev/null +++ b/share/default/config/tracker.container.postgresql.toml @@ -0,0 +1,32 @@ +[metadata] +app = "torrust-tracker" +purpose = "configuration" +schema_version = "2.0.0" + +[logging] +threshold = "info" + +[core] +listed = false +private = false + +[core.database] +driver = "postgresql" +# If the PostgreSQL password includes reserved URL characters (for example + or /), +# percent-encode it in the DSN password component. +# Example: password a+b/c -> a%2Bb%2Fc +path = "postgresql://postgres:postgres@postgres:5432/torrust_tracker" + +# Uncomment to enable services + +#[[udp_trackers]] +#bind_address = "0.0.0.0:6969" + +#[[http_trackers]] +#bind_address = "0.0.0.0:7070" + +#[http_api] +#bind_address = "0.0.0.0:1212" + +#[http_api.access_tokens] +#admin = "MyAccessToken" From 15af1e078cf1441ca0b89d9f456be3bd2b4e892a Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 09:57:37 +0100 Subject: [PATCH 04/22] fix(tracker-core): correct postgres key timestamp column --- .../20240730183000_torrust_tracker_create_all_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql b/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql index 509cb97ff..ee6291303 100644 --- a/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql +++ b/packages/tracker-core/migrations/postgresql/20240730183000_torrust_tracker_create_all_tables.sql @@ -16,5 +16,5 @@ CREATE TABLE IF NOT EXISTS keys ( id SERIAL PRIMARY KEY, key VARCHAR(32) NOT NULL UNIQUE, - valid_until INTEGER NOT NULL + valid_until BIGINT NOT NULL ); \ No newline at end of file From 54210f3f6e6557e97230cce1c8fa837c328d86ce Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 09:58:14 +0100 Subject: [PATCH 05/22] ci(testing): add postgres compatibility job --- .github/workflows/testing.yaml | 37 +++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 0d5753e5d..ef4e1b0b7 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -133,8 +133,8 @@ jobs: name: Run Unit Tests run: cargo test --tests --benches --examples --workspace --all-targets --all-features - database-compatibility: - name: Database Compatibility (${{ matrix.mysql-version }}) + database-compatibility-mysql: + name: Database Compatibility MySQL (${{ matrix.mysql-version }}) runs-on: ubuntu-latest needs: unit @@ -164,10 +164,41 @@ jobs: TORRUST_TRACKER_CORE_MYSQL_DRIVER_IMAGE_TAG: ${{ matrix.mysql-version }} run: cargo test -p bittorrent-tracker-core --features db-compatibility-tests run_mysql_driver_tests -- --nocapture + database-compatibility-postgres: + name: Database Compatibility PostgreSQL (${{ matrix.postgres-version }}) + runs-on: ubuntu-latest + needs: unit + + strategy: + matrix: + postgres-version: ["15", "16", "17"] + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: database + name: Run Database Compatibility Test + env: + TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST: "true" + TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG: ${{ matrix.postgres-version }} + run: cargo test -p bittorrent-tracker-core --features db-compatibility-tests run_postgres_driver_tests -- --nocapture + e2e: name: E2E runs-on: ubuntu-latest - needs: database-compatibility + needs: [database-compatibility-mysql, database-compatibility-postgres] timeout-minutes: 45 strategy: From 74f5c8a9305912db8873024156cc006662ad1902 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 10:44:44 +0100 Subject: [PATCH 06/22] feat(ci): extend qBittorrent E2E runner with MySQL and PostgreSQL - Rename compose.qbittorrent-e2e.yaml to compose.qbittorrent-e2e.sqlite3.yaml - Add compose.qbittorrent-e2e.mysql.yaml for MySQL backend E2E tests - Add compose.qbittorrent-e2e.postgresql.yaml for PostgreSQL backend E2E tests - Add --db-driver CLI argument to qBittorrent E2E runner for backend selection - Select compose file and generate tracker config based on chosen backend - Add MySQL and PostgreSQL qBittorrent E2E CI steps to testing.yaml - Update SQLite CI step to use renamed compose file path - Update qbt debug scripts and README to use renamed compose file --- .github/workflows/testing.yaml | 14 ++- compose.qbittorrent-e2e.mysql.yaml | 88 +++++++++++++++++++ compose.qbittorrent-e2e.postgresql.yaml | 85 ++++++++++++++++++ ...ml => compose.qbittorrent-e2e.sqlite3.yaml | 0 contrib/dev-tools/debugging/qbt/README.md | 2 +- .../qbt/check-qbittorrent-e2e-compose.sh | 2 +- src/bin/qbittorrent_e2e_runner.rs | 11 +-- src/console/ci/qbittorrent_e2e/mod.rs | 2 +- src/console/ci/qbittorrent_e2e/runner.rs | 54 ++++++++++-- .../qbittorrent_e2e/tracker/config_builder.rs | 44 ++++++++-- src/console/ci/qbittorrent_e2e/tracker/mod.rs | 2 +- 11 files changed, 281 insertions(+), 23 deletions(-) create mode 100644 compose.qbittorrent-e2e.mysql.yaml create mode 100644 compose.qbittorrent-e2e.postgresql.yaml rename compose.qbittorrent-e2e.yaml => compose.qbittorrent-e2e.sqlite3.yaml (100%) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index ef4e1b0b7..b07d6267a 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -227,5 +227,15 @@ jobs: - id: run-qbittorrent-e2e-test if: matrix.toolchain == 'stable' - name: Run qBittorrent E2E Test - run: cargo run --bin qbittorrent_e2e_runner -- --compose-file ./compose.qbittorrent-e2e.yaml --timeout-seconds 600 + name: Run qBittorrent E2E Test (SQLite) + run: cargo run --bin qbittorrent_e2e_runner -- --compose-file ./compose.qbittorrent-e2e.sqlite3.yaml --timeout-seconds 600 + + - id: run-qbittorrent-e2e-test-mysql + if: matrix.toolchain == 'stable' + name: Run qBittorrent E2E Test (MySQL) + run: cargo run --bin qbittorrent_e2e_runner -- --db-driver mysql --timeout-seconds 600 + + - id: run-qbittorrent-e2e-test-postgresql + if: matrix.toolchain == 'stable' + name: Run qBittorrent E2E Test (PostgreSQL) + run: cargo run --bin qbittorrent_e2e_runner -- --db-driver postgresql --timeout-seconds 600 diff --git a/compose.qbittorrent-e2e.mysql.yaml b/compose.qbittorrent-e2e.mysql.yaml new file mode 100644 index 000000000..fd783a958 --- /dev/null +++ b/compose.qbittorrent-e2e.mysql.yaml @@ -0,0 +1,88 @@ +name: qbittorrent-e2e + +services: + tracker: + build: + context: . + dockerfile: Containerfile + target: release + image: ${QBT_E2E_TRACKER_IMAGE:?QBT_E2E_TRACKER_IMAGE is required} + restart: "no" + environment: + TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER: mysql + depends_on: + mysql: + condition: service_healthy + volumes: + - type: bind + source: ${QBT_E2E_TRACKER_CONFIG_PATH:?QBT_E2E_TRACKER_CONFIG_PATH is required} + target: /etc/torrust/tracker/tracker.toml + read_only: true + - type: bind + source: ${QBT_E2E_TRACKER_STORAGE_PATH:?QBT_E2E_TRACKER_STORAGE_PATH is required} + target: /var/lib/torrust/tracker + ports: + - "0:${QBT_E2E_TRACKER_HTTP_TRACKER_PORT:?QBT_E2E_TRACKER_HTTP_TRACKER_PORT is required}" + - "0:${QBT_E2E_TRACKER_UDP_PORT:?QBT_E2E_TRACKER_UDP_PORT is required}/udp" + - "0:${QBT_E2E_TRACKER_HTTP_API_PORT:?QBT_E2E_TRACKER_HTTP_API_PORT is required}" + - "0:${QBT_E2E_TRACKER_HEALTH_CHECK_API_PORT:?QBT_E2E_TRACKER_HEALTH_CHECK_API_PORT is required}" + + mysql: + image: mysql:8.0 + command: "--default-authentication-plugin=mysql_native_password" + restart: "no" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -proot_secret_password --silent"] + interval: 3s + retries: 20 + start_period: 20s + environment: + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: root_secret_password + MYSQL_DATABASE: torrust_tracker + MYSQL_USER: db_user + MYSQL_PASSWORD: db_user_secret_password + + qbittorrent-seeder: + image: ${QBT_E2E_QBITTORRENT_IMAGE:?QBT_E2E_QBITTORRENT_IMAGE is required} + restart: "no" + environment: + WEBUI_PORT: "8080" + PUID: "1000" + PGID: "1000" + TZ: "UTC" + QBT_LEGAL_NOTICE: "confirm" + volumes: + - type: bind + source: ${QBT_E2E_SEEDER_CONFIG_PATH:?QBT_E2E_SEEDER_CONFIG_PATH is required} + target: /config + - type: bind + source: ${QBT_E2E_SEEDER_DOWNLOADS_PATH:?QBT_E2E_SEEDER_DOWNLOADS_PATH is required} + target: /downloads + - type: bind + source: ${QBT_E2E_SHARED_PATH:?QBT_E2E_SHARED_PATH is required} + target: /shared + ports: + - "0:8080" + + qbittorrent-leecher: + image: ${QBT_E2E_QBITTORRENT_IMAGE:?QBT_E2E_QBITTORRENT_IMAGE is required} + restart: "no" + environment: + WEBUI_PORT: "8080" + PUID: "1000" + PGID: "1000" + TZ: "UTC" + QBT_LEGAL_NOTICE: "confirm" + volumes: + - type: bind + source: ${QBT_E2E_LEECHER_CONFIG_PATH:?QBT_E2E_LEECHER_CONFIG_PATH is required} + target: /config + - type: bind + source: ${QBT_E2E_LEECHER_DOWNLOADS_PATH:?QBT_E2E_LEECHER_DOWNLOADS_PATH is required} + target: /downloads + - type: bind + source: ${QBT_E2E_SHARED_PATH:?QBT_E2E_SHARED_PATH is required} + target: /shared + ports: + - "0:8080" diff --git a/compose.qbittorrent-e2e.postgresql.yaml b/compose.qbittorrent-e2e.postgresql.yaml new file mode 100644 index 000000000..d5131820c --- /dev/null +++ b/compose.qbittorrent-e2e.postgresql.yaml @@ -0,0 +1,85 @@ +name: qbittorrent-e2e + +services: + tracker: + build: + context: . + dockerfile: Containerfile + target: release + image: ${QBT_E2E_TRACKER_IMAGE:?QBT_E2E_TRACKER_IMAGE is required} + restart: "no" + environment: + TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER: postgresql + depends_on: + postgres: + condition: service_healthy + volumes: + - type: bind + source: ${QBT_E2E_TRACKER_CONFIG_PATH:?QBT_E2E_TRACKER_CONFIG_PATH is required} + target: /etc/torrust/tracker/tracker.toml + read_only: true + - type: bind + source: ${QBT_E2E_TRACKER_STORAGE_PATH:?QBT_E2E_TRACKER_STORAGE_PATH is required} + target: /var/lib/torrust/tracker + ports: + - "0:${QBT_E2E_TRACKER_HTTP_TRACKER_PORT:?QBT_E2E_TRACKER_HTTP_TRACKER_PORT is required}" + - "0:${QBT_E2E_TRACKER_UDP_PORT:?QBT_E2E_TRACKER_UDP_PORT is required}/udp" + - "0:${QBT_E2E_TRACKER_HTTP_API_PORT:?QBT_E2E_TRACKER_HTTP_API_PORT is required}" + - "0:${QBT_E2E_TRACKER_HEALTH_CHECK_API_PORT:?QBT_E2E_TRACKER_HEALTH_CHECK_API_PORT is required}" + + postgres: + image: postgres:17 + restart: "no" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d torrust_tracker"] + interval: 3s + retries: 20 + start_period: 10s + environment: + POSTGRES_DB: torrust_tracker + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + + qbittorrent-seeder: + image: ${QBT_E2E_QBITTORRENT_IMAGE:?QBT_E2E_QBITTORRENT_IMAGE is required} + restart: "no" + environment: + WEBUI_PORT: "8080" + PUID: "1000" + PGID: "1000" + TZ: "UTC" + QBT_LEGAL_NOTICE: "confirm" + volumes: + - type: bind + source: ${QBT_E2E_SEEDER_CONFIG_PATH:?QBT_E2E_SEEDER_CONFIG_PATH is required} + target: /config + - type: bind + source: ${QBT_E2E_SEEDER_DOWNLOADS_PATH:?QBT_E2E_SEEDER_DOWNLOADS_PATH is required} + target: /downloads + - type: bind + source: ${QBT_E2E_SHARED_PATH:?QBT_E2E_SHARED_PATH is required} + target: /shared + ports: + - "0:8080" + + qbittorrent-leecher: + image: ${QBT_E2E_QBITTORRENT_IMAGE:?QBT_E2E_QBITTORRENT_IMAGE is required} + restart: "no" + environment: + WEBUI_PORT: "8080" + PUID: "1000" + PGID: "1000" + TZ: "UTC" + QBT_LEGAL_NOTICE: "confirm" + volumes: + - type: bind + source: ${QBT_E2E_LEECHER_CONFIG_PATH:?QBT_E2E_LEECHER_CONFIG_PATH is required} + target: /config + - type: bind + source: ${QBT_E2E_LEECHER_DOWNLOADS_PATH:?QBT_E2E_LEECHER_DOWNLOADS_PATH is required} + target: /downloads + - type: bind + source: ${QBT_E2E_SHARED_PATH:?QBT_E2E_SHARED_PATH is required} + target: /shared + ports: + - "0:8080" diff --git a/compose.qbittorrent-e2e.yaml b/compose.qbittorrent-e2e.sqlite3.yaml similarity index 100% rename from compose.qbittorrent-e2e.yaml rename to compose.qbittorrent-e2e.sqlite3.yaml diff --git a/contrib/dev-tools/debugging/qbt/README.md b/contrib/dev-tools/debugging/qbt/README.md index 1f8507f96..f989742db 100644 --- a/contrib/dev-tools/debugging/qbt/README.md +++ b/contrib/dev-tools/debugging/qbt/README.md @@ -45,7 +45,7 @@ How to verify: 1. Confirm the leecher port mapping. 2. Compare login responses with and without host header override. - docker compose -f ./compose.qbittorrent-e2e.yaml -p port qbittorrent-leecher 8080 + docker compose -f ./compose.qbittorrent-e2e.sqlite3.yaml -p port qbittorrent-leecher 8080 curl -i -X POST http://127.0.0.1:/api/v2/auth/login \ --data 'username=admin&password=adminadmin' curl -i -X POST http://127.0.0.1:/api/v2/auth/login \ diff --git a/contrib/dev-tools/debugging/qbt/check-qbittorrent-e2e-compose.sh b/contrib/dev-tools/debugging/qbt/check-qbittorrent-e2e-compose.sh index ce57b1066..b7ac8a4c3 100755 --- a/contrib/dev-tools/debugging/qbt/check-qbittorrent-e2e-compose.sh +++ b/contrib/dev-tools/debugging/qbt/check-qbittorrent-e2e-compose.sh @@ -5,7 +5,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" -COMPOSE_FILE="$REPO_ROOT/compose.qbittorrent-e2e.yaml" +COMPOSE_FILE="$REPO_ROOT/compose.qbittorrent-e2e.sqlite3.yaml" TRACKER_IMAGE="torrust-tracker:qbt-e2e-local" QBITTORRENT_IMAGE="lscr.io/linuxserver/qbittorrent:5.1.4" PROJECT_NAME="qbt-e2e-composecheck-$(date +%s)" diff --git a/src/bin/qbittorrent_e2e_runner.rs b/src/bin/qbittorrent_e2e_runner.rs index e8017a041..973e6d0b0 100644 --- a/src/bin/qbittorrent_e2e_runner.rs +++ b/src/bin/qbittorrent_e2e_runner.rs @@ -6,9 +6,9 @@ //! 1. Builds a local Torrust Tracker Docker image. //! 2. Creates an ephemeral workspace (temporary directory) with all required //! configuration files and pre-generated torrent + payload. -//! 3. Starts a Docker Compose stack (`compose.qbittorrent-e2e.yaml`) containing -//! a tracker, a seeder, and a leecher — all using randomly assigned host ports -//! so multiple runs can coexist. +//! 3. Starts a backend-specific Docker Compose stack containing a tracker, a +//! seeder, and a leecher. The default stack is `SQLite`, while `--db-driver` +//! can switch to `MySQL` or `PostgreSQL`. //! 4. Authenticates with both `qBittorrent` `WebUI` instances. //! 5. Uploads the torrent to the seeder and the leecher. //! 6. Logs the torrent count reported by each client. @@ -25,7 +25,7 @@ //! //! ```text //! cargo run --bin qbittorrent_e2e_runner -- \ -//! --compose-file ./compose.qbittorrent-e2e.yaml \ +//! --db-driver postgresql \ //! --timeout-seconds 180 //! ``` //! @@ -33,7 +33,8 @@ //! //! | Flag | Default | Description | //! |------|---------|-------------| -//! | `--compose-file` | `compose.qbittorrent-e2e.yaml` | Compose file for the scenario | +//! | `--db-driver` | `sqlite3` | Tracker database backend: `sqlite3`, `mysql`, or `postgresql` | +//! | `--compose-file` | driver-specific default | Override the compose file selected for the scenario | //! | `--timeout-seconds` | `180` | Per-operation HTTP timeout for `WebUI` calls | //! | `--tracker-image` | `torrust-tracker:qbt-e2e-local` | Local Docker image tag built for the tracker | //! | `--qbittorrent-image` | `lscr.io/linuxserver/qbittorrent:5.1.4` | qBittorrent image for seeder and leecher | diff --git a/src/console/ci/qbittorrent_e2e/mod.rs b/src/console/ci/qbittorrent_e2e/mod.rs index 2a006d38e..e20e2c4e8 100644 --- a/src/console/ci/qbittorrent_e2e/mod.rs +++ b/src/console/ci/qbittorrent_e2e/mod.rs @@ -52,7 +52,7 @@ //! //! This also opens a clear extension path: in the future we could have multiple //! infrastructure configurations (e.g. public vs. private tracker, `SQLite` vs. -//! `MySQL`, different numbers of peers) each hosting their own suite of scenarios, +//! `MySQL` vs. `PostgreSQL`, different numbers of peers) each hosting their own suite of scenarios, //! without changing the scenario or step code. pub mod bencode; diff --git a/src/console/ci/qbittorrent_e2e/runner.rs b/src/console/ci/qbittorrent_e2e/runner.rs index 441ad0992..3df18d581 100644 --- a/src/console/ci/qbittorrent_e2e/runner.rs +++ b/src/console/ci/qbittorrent_e2e/runner.rs @@ -3,27 +3,63 @@ //! Example: //! //! ```text -//! cargo run --bin qbittorrent_e2e_runner -- --compose-file ./compose.qbittorrent-e2e.yaml --timeout-seconds 300 +//! cargo run --bin qbittorrent_e2e_runner -- --db-driver postgresql --timeout-seconds 300 //! ``` use std::path::PathBuf; use std::time::Duration; -use clap::Parser; +use clap::{Parser, ValueEnum}; use tracing::level_filters::LevelFilter; -use super::tracker::TrackerConfig; +use super::tracker::{DatabaseDriver, TrackerConfig}; use super::types::{ComposeProjectName, QbittorrentImage, TrackerImage}; use super::{filesystem_setup, scenarios, services_setup}; +const SQLITE3_COMPOSE_FILE: &str = "compose.qbittorrent-e2e.sqlite3.yaml"; +const MYSQL_COMPOSE_FILE: &str = "compose.qbittorrent-e2e.mysql.yaml"; +const POSTGRESQL_COMPOSE_FILE: &str = "compose.qbittorrent-e2e.postgresql.yaml"; const TRACKER_IMAGE: &str = "torrust-tracker:qbt-e2e-local"; const QBITTORRENT_IMAGE: &str = "lscr.io/linuxserver/qbittorrent:5.1.4"; +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum DbDriverArg { + #[value(name = "sqlite3")] + Sqlite3, + #[value(name = "mysql")] + MySQL, + #[value(name = "postgresql")] + PostgreSQL, +} + +impl DbDriverArg { + fn default_compose_file(self) -> &'static str { + match self { + Self::Sqlite3 => SQLITE3_COMPOSE_FILE, + Self::MySQL => MYSQL_COMPOSE_FILE, + Self::PostgreSQL => POSTGRESQL_COMPOSE_FILE, + } + } + + fn database_driver(self) -> DatabaseDriver { + match self { + Self::Sqlite3 => DatabaseDriver::Sqlite3, + Self::MySQL => DatabaseDriver::MySQL, + Self::PostgreSQL => DatabaseDriver::PostgreSQL, + } + } +} + #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Args { + /// Database backend used by the tracker container. + #[clap(long, value_enum, default_value_t = DbDriverArg::Sqlite3)] + db_driver: DbDriverArg, + /// Compose file used for the qBittorrent scenario. - #[clap(long, default_value = "compose.qbittorrent-e2e.yaml")] - compose_file: PathBuf, + /// Defaults to a backend-specific scenario file when omitted. + #[clap(long)] + compose_file: Option, /// Timeout in seconds for API operations. #[clap(long, default_value_t = 180)] @@ -56,11 +92,15 @@ pub async fn run() -> anyhow::Result<()> { tracing_stdout_init(LevelFilter::INFO); let args = Args::parse(); + let compose_file = args + .compose_file + .clone() + .unwrap_or_else(|| PathBuf::from(args.db_driver.default_compose_file())); let project_name = ComposeProjectName::generate(&args.project_prefix); tracing::info!("Using compose project name: {project_name}"); let timeout = Duration::from_secs(args.timeout_seconds); - let tracker_config = TrackerConfig::default(); + let tracker_config = TrackerConfig::for_database_driver(args.db_driver.database_driver()); let workspace = filesystem_setup::prepare(&project_name, args.keep_containers, timeout, &tracker_config)?; let resources = workspace.resources(); @@ -70,7 +110,7 @@ pub async fn run() -> anyhow::Result<()> { let qbittorrent_image = QbittorrentImage::new(&args.qbittorrent_image); let (mut running_compose, seeder, leecher, tracker) = services_setup::start( - &args.compose_file, + &compose_file, &project_name, &tracker_image, &qbittorrent_image, diff --git a/src/console/ci/qbittorrent_e2e/tracker/config_builder.rs b/src/console/ci/qbittorrent_e2e/tracker/config_builder.rs index 157a8e0c0..086d186ba 100644 --- a/src/console/ci/qbittorrent_e2e/tracker/config_builder.rs +++ b/src/console/ci/qbittorrent_e2e/tracker/config_builder.rs @@ -4,10 +4,12 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use anyhow::Context; -use torrust_tracker_configuration::{Configuration, HealthCheckApi, HttpApi, HttpTracker, UdpTracker}; +use torrust_tracker_configuration::{Configuration, Driver, HealthCheckApi, HttpApi, HttpTracker, UdpTracker}; const CONFIG_FILE_NAME: &str = "tracker-config.toml"; -const DEFAULT_DATABASE_PATH: &str = "/var/lib/torrust/tracker/database/sqlite3.db"; +const DEFAULT_SQLITE3_DATABASE_PATH: &str = "/var/lib/torrust/tracker/database/sqlite3.db"; +const DEFAULT_MYSQL_DATABASE_PATH: &str = "mysql://db_user:db_user_secret_password@mysql:3306/torrust_tracker"; +const DEFAULT_POSTGRESQL_DATABASE_PATH: &str = "postgresql://postgres:postgres@postgres:5432/torrust_tracker"; const TRACKER_BIND_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED); const TRACKER_UDP_PORT: u16 = 6969; const TRACKER_HTTP_TRACKER_PORT: u16 = 7070; @@ -15,9 +17,35 @@ const TRACKER_HTTP_API_PORT: u16 = 1212; const TRACKER_HEALTH_CHECK_API_PORT: u16 = 1313; const DEFAULT_ACCESS_TOKEN: &str = "MyAccessToken"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DatabaseDriver { + Sqlite3, + MySQL, + PostgreSQL, +} + +impl DatabaseDriver { + fn configuration_driver(self) -> Driver { + match self { + Self::Sqlite3 => Driver::Sqlite3, + Self::MySQL => Driver::MySQL, + Self::PostgreSQL => Driver::PostgreSQL, + } + } + + fn default_database_path(self) -> &'static str { + match self { + Self::Sqlite3 => DEFAULT_SQLITE3_DATABASE_PATH, + Self::MySQL => DEFAULT_MYSQL_DATABASE_PATH, + Self::PostgreSQL => DEFAULT_POSTGRESQL_DATABASE_PATH, + } + } +} + /// Typed tracker configuration shared across the E2E workflow. #[derive(Clone, Debug)] pub(crate) struct TrackerConfig { + database_driver: DatabaseDriver, database_path: String, udp_bind_address: SocketAddr, http_tracker_bind_address: SocketAddr, @@ -28,8 +56,15 @@ pub(crate) struct TrackerConfig { impl Default for TrackerConfig { fn default() -> Self { + Self::for_database_driver(DatabaseDriver::Sqlite3) + } +} + +impl TrackerConfig { + pub(crate) fn for_database_driver(database_driver: DatabaseDriver) -> Self { Self { - database_path: DEFAULT_DATABASE_PATH.to_string(), + database_driver, + database_path: database_driver.default_database_path().to_string(), udp_bind_address: bind_address(TRACKER_UDP_PORT), http_tracker_bind_address: bind_address(TRACKER_HTTP_TRACKER_PORT), http_api_bind_address: bind_address(TRACKER_HTTP_API_PORT), @@ -37,9 +72,7 @@ impl Default for TrackerConfig { access_token: DEFAULT_ACCESS_TOKEN.to_string(), } } -} -impl TrackerConfig { pub(crate) fn udp_bind_address(&self) -> SocketAddr { self.udp_bind_address } @@ -73,6 +106,7 @@ impl TrackerConfig { fn to_torrust_configuration(&self) -> Configuration { let mut configuration = Configuration::default(); + configuration.core.database.driver = self.database_driver.configuration_driver(); configuration.core.database.path.clone_from(&self.database_path); configuration.udp_trackers = Some(vec![UdpTracker { diff --git a/src/console/ci/qbittorrent_e2e/tracker/mod.rs b/src/console/ci/qbittorrent_e2e/tracker/mod.rs index 10b6e2a1d..d887a3d60 100644 --- a/src/console/ci/qbittorrent_e2e/tracker/mod.rs +++ b/src/console/ci/qbittorrent_e2e/tracker/mod.rs @@ -3,4 +3,4 @@ mod client; mod config_builder; pub(crate) use client::TrackerApiClient; -pub(super) use config_builder::{TrackerConfig, TrackerConfigBuilder}; +pub(super) use config_builder::{DatabaseDriver, TrackerConfig, TrackerConfigBuilder}; From e0d0a8729f0edd47c5a3fc8893e9314ed6a9006c Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:10:28 +0100 Subject: [PATCH 07/22] fix(tracker-core): wait for single postgres ready log in benchmark runner --- .../driver_bench/database/postgres.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs index 62d79df33..b3530e2eb 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/driver_bench/database/postgres.rs @@ -19,11 +19,16 @@ const READINESS_PING_RETRIES: usize = 30; const READINESS_PING_INTERVAL: Duration = Duration::from_millis(500); pub(super) async fn initialize(db_version: &str) -> Result { + // The official `postgres` image emits "database system is ready to accept + // connections" once on stderr when the TCP listener is up. We wait for + // that single occurrence before probing the connection — this mirrors the + // two-occurrence strategy used for MySQL where the init cycle emits it + // twice. PostgreSQL only emits it once. let postgres_container = GenericImage::new("postgres", db_version) .with_exposed_port(5432.tcp()) - .with_wait_for(WaitFor::Log( - LogWaitStrategy::stderr("database system is ready to accept connections").with_times(2), - )) + .with_wait_for(WaitFor::Log(LogWaitStrategy::stderr( + "database system is ready to accept connections", + ))) .with_env_var("POSTGRES_PASSWORD", "test") .with_env_var("POSTGRES_DB", "torrust_tracker_bench") .with_env_var("POSTGRES_USER", "root") From aee2efbefd3842a2c13551d884cf8ce120616f8b Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:15:26 +0100 Subject: [PATCH 08/22] docs(tracker-core): add 2026-05-01 benchmark run with PostgreSQL baseline --- .../tracker-core/docs/benchmarking/README.md | 30 ++++- .../machine/2026-05-01-josecelano-desktop.txt | 96 ++++++++++++++ .../benchmarking/runs/2026-05-01/REPORT.md | 85 ++++++++++++ .../runs/2026-05-01/mysql-8.0.json | 121 ++++++++++++++++++ .../runs/2026-05-01/mysql-8.4.json | 121 ++++++++++++++++++ .../runs/2026-05-01/postgresql-17.json | 121 ++++++++++++++++++ .../benchmarking/runs/2026-05-01/sqlite3.json | 121 ++++++++++++++++++ 7 files changed, 689 insertions(+), 6 deletions(-) create mode 100644 packages/tracker-core/docs/benchmarking/machine/2026-05-01-josecelano-desktop.txt create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-05-01/REPORT.md create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.0.json create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.4.json create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-05-01/postgresql-17.json create mode 100644 packages/tracker-core/docs/benchmarking/runs/2026-05-01/sqlite3.json diff --git a/packages/tracker-core/docs/benchmarking/README.md b/packages/tracker-core/docs/benchmarking/README.md index b3b5af704..5d19d9e49 100644 --- a/packages/tracker-core/docs/benchmarking/README.md +++ b/packages/tracker-core/docs/benchmarking/README.md @@ -28,7 +28,7 @@ Raw JSON artifacts: - `runs/2026-04-28/mysql-8.4.json` - `runs/2026-04-28/mysql-8.0.json` -## Post-SQLx run +## Post-SQLx run (SQLite and MySQL only) - Date: `2026-04-30` - Commit (HEAD at run time): `a4dbc63a6c713e115bfc11374b72743aa51ebfb5` @@ -42,6 +42,21 @@ Raw JSON artifacts: - `runs/2026-04-30/mysql-8.4.json` - `runs/2026-04-30/mysql-8.0.json` +## PostgreSQL baseline run + +- Date: `2026-05-01` +- Commit (HEAD at run time): `74f5c8a9305912db8873024156cc006662ad1902` +- Issue context: `docs/issues/1723-1525-08-add-postgresql-driver.md` +- Run summary (first run with PostgreSQL): `runs/2026-05-01/REPORT.md` +- Machine profile: `machine/2026-05-01-josecelano-desktop.txt` + +Raw JSON artifacts: + +- `runs/2026-05-01/sqlite3.json` +- `runs/2026-05-01/mysql-8.4.json` +- `runs/2026-05-01/mysql-8.0.json` +- `runs/2026-05-01/postgresql-17.json` + ## How to add a new run 1. Create a new run folder: @@ -54,6 +69,8 @@ Raw JSON artifacts: `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4 > packages/tracker-core/docs/benchmarking/runs/YYYY-MM-DD/mysql-8.4.json` + `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver postgresql --db-version 17 > packages/tracker-core/docs/benchmarking/runs/YYYY-MM-DD/postgresql-17.json` + 3. Capture machine profile: `mkdir -p packages/tracker-core/docs/benchmarking/machine` @@ -72,10 +89,11 @@ Raw JSON artifacts: ## Planned comparison point -After implementing: - -- `docs/issues/1525-05-migrate-sqlite-and-mysql-to-sqlx.md` +After implementing `docs/issues/1717-1525-05-migrate-sqlite-and-mysql-to-sqlx.md`, the +benchmark was re-run at `runs/2026-04-30` to compare against the `2026-04-28` baseline. -run the same benchmark commands again, store results in a new dated folder, and compare against `runs/2026-04-28`. +After adding the PostgreSQL driver (`docs/issues/1723-1525-08-add-postgresql-driver.md`), +the benchmark was run again at `runs/2026-05-01` to establish the PostgreSQL baseline. -The first such comparison was captured at `runs/2026-04-30/REPORT.md`. +The next planned comparison point is after any major persistence refactor that touches all +drivers (e.g., schema migrations or async `sqlx` pool changes). diff --git a/packages/tracker-core/docs/benchmarking/machine/2026-05-01-josecelano-desktop.txt b/packages/tracker-core/docs/benchmarking/machine/2026-05-01-josecelano-desktop.txt new file mode 100644 index 000000000..55cac57de --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/machine/2026-05-01-josecelano-desktop.txt @@ -0,0 +1,96 @@ +hostname: +josecelano-desktop + +date_utc: +2026-05-01T10:10:57Z + +uname -a: +Linux josecelano-desktop 6.17.0-22-generic #22-Ubuntu SMP PREEMPT_DYNAMIC Fri Mar 13 12:04:44 UTC 2026 x86_64 GNU/Linux + +/etc/os-release: +PRETTY_NAME="Ubuntu 25.10" +NAME="Ubuntu" +VERSION_ID="25.10" +VERSION="25.10 (Questing Quokka)" +VERSION_CODENAME=questing +ID=ubuntu +ID_LIKE=debian +HOME_URL="https://www.ubuntu.com/" +SUPPORT_URL="https://help.ubuntu.com/" +BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" +PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" +UBUNTU_CODENAME=questing +LOGO=ubuntu-logo + +lscpu: +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Address sizes: 48 bits physical, 48 bits virtual +Byte Order: Little Endian +CPU(s): 32 +On-line CPU(s) list: 0-31 +Vendor ID: AuthenticAMD +Model name: AMD Ryzen 9 7950X 16-Core Processor +CPU family: 25 +Model: 97 +Thread(s) per core: 2 +Core(s) per socket: 16 +Socket(s): 1 +Stepping: 2 +Frequency boost: enabled +CPU(s) scaling MHz: 74% +CPU max MHz: 5883,1968 +CPU min MHz: 425,2920 +BogoMIPS: 8982,52 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpuid_fault cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d amd_lbr_pmc_freeze +Virtualization: AMD-V +L1d cache: 512 KiB (16 instances) +L1i cache: 512 KiB (16 instances) +L2 cache: 16 MiB (16 instances) +L3 cache: 64 MiB (2 instances) +NUMA node(s): 1 +NUMA node0 CPU(s): 0-31 +Vulnerability Gather data sampling: Not affected +Vulnerability Ghostwrite: Not affected +Vulnerability Indirect target selection: Not affected +Vulnerability Itlb multihit: Not affected +Vulnerability L1tf: Not affected +Vulnerability Mds: Not affected +Vulnerability Meltdown: Not affected +Vulnerability Mmio stale data: Not affected +Vulnerability Old microcode: Not affected +Vulnerability Reg file data sampling: Not affected +Vulnerability Retbleed: Not affected +Vulnerability Spec rstack overflow: Mitigation; Safe RET +Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl +Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization +Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; STIBP always-on; PBRSB-eIBRS Not affected; BHI Not affected +Vulnerability Srbds: Not affected +Vulnerability Tsa: Mitigation; Clear CPU buffers +Vulnerability Tsx async abort: Not affected +Vulnerability Vmscape: Mitigation; IBPB before exit to userspace + +free -h: + total used free shared buff/cache available +Mem: 61Gi 16Gi 31Gi 324Mi 13Gi 44Gi +Swap: 8,0Gi 5,5Gi 2,5Gi + +docker --version: +Docker version 28.3.3, build 980b856 + +rustup show: +Default host: x86_64-unknown-linux-gnu +rustup home: /home/josecelano/.rustup + +installed toolchains +-------------------- +stable-x86_64-unknown-linux-gnu +nightly-x86_64-unknown-linux-gnu (active, default) +1.74.0-x86_64-unknown-linux-gnu + +active toolchain +---------------- +name: nightly-x86_64-unknown-linux-gnu +active because: it's the default toolchain +installed targets: + x86_64-unknown-linux-gnu diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-05-01/REPORT.md b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/REPORT.md new file mode 100644 index 000000000..143759399 --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/REPORT.md @@ -0,0 +1,85 @@ +# Benchmark Report - 2026-05-01 + +This run captures the first benchmark results that include a PostgreSQL driver, +added in subissue #1525-08: + +- `docs/issues/1723-1525-08-add-postgresql-driver.md` + +It is the first run to exercise `--driver postgresql` and establishes the +PostgreSQL baseline alongside the existing SQLite and MySQL numbers. + +## Run context + +- Commit (HEAD at run time): `74f5c8a9305912db8873024156cc006662ad1902` +- Ops per operation: `100` +- Benchmark runner: `cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner` +- Machine profile: `../../machine/2026-05-01-josecelano-desktop.txt` +- Same machine as all prior runs (AMD Ryzen 9 7950X, Ubuntu 25.10). + +## Raw artifacts + +- `sqlite3.json` +- `mysql-8.4.json` +- `mysql-8.0.json` +- `postgresql-17.json` + +## High-level timing summary + +`meta.timings_ms.total`: + +| Driver | 2026-04-30 | 2026-05-01 | Delta | +| ------------- | ---------: | ---------: | ------: | +| sqlite3 | 118 ms | 119 ms | +1 ms | +| mysql 8.4 | 6231 ms | 6372 ms | +141 ms | +| mysql 8.0 | 6678 ms | 7272 ms | +594 ms | +| postgresql 17 | — | 1451 ms | — | + +Note: SQLite and MySQL totals are stable and within run-to-run noise. +PostgreSQL 17 is new in this run — no prior baseline to compare against. + +## Selected operation medians (microseconds) + +| Operation | sqlite3 | mysql 8.4 | mysql 8.0 | postgresql 17 | +| ------------------------------- | ------: | --------: | --------: | ------------: | +| save_torrent_downloads | 89 | 769 | 984 | 298 | +| load_torrent_downloads | 23 | 112 | 115 | 88 | +| load_all_torrents_downloads | 77 | 172 | 171 | 146 | +| increase_downloads_for_torrent | 70 | 773 | 1005 | 302 | +| save_global_downloads | 76 | 793 | 1066 | 299 | +| load_global_downloads | 21 | 115 | 137 | 86 | +| increase_global_downloads | 67 | 774 | 1036 | 305 | +| add_info_hash_to_whitelist | 81 | 735 | 981 | 294 | +| get_info_hash_from_whitelist | 21 | 109 | 118 | 95 | +| load_whitelist | 55 | 161 | 175 | 135 | +| remove_info_hash_from_whitelist | 81 | 766 | 962 | 293 | +| add_key_to_keys | 81 | 750 | 974 | 292 | +| get_key_from_keys | 22 | 118 | 129 | 95 | +| load_keys | 77 | 167 | 189 | 155 | +| remove_key_from_keys | 73 | 739 | 994 | 300 | + +## PostgreSQL 17 characteristics + +- Write operations (`save_*`, `increase_*`, `add_*`, `remove_*`): median ~290–305 µs. + Roughly 2.5–3× faster than MySQL 8.0 and ~60% faster than MySQL 8.4 for writes. +- Read operations (`load_*`, `get_*`): median 86–155 µs. + Comparable to MySQL 8.4 for simple lookups; slightly slower for `load_*` aggregates. +- Overall total (1451 ms) is significantly lower than both MySQL versions, driven by + faster write operations. +- `remove_*` operations (293–300 µs) are notably faster than MySQL (739–994 µs). + +## Regression assessment + +No regression. SQLite and MySQL numbers are within noise of the `2026-04-30` run. +PostgreSQL 17 is introduced as a new baseline — no comparison is possible yet. + +## Machine characteristics (summary) + +From `../../machine/2026-05-01-josecelano-desktop.txt`: + +- Host: `josecelano-desktop` +- OS: `Ubuntu 25.10` +- Kernel: `Linux 6.17.0-22-generic` +- CPU: `AMD Ryzen 9 7950X` (16 cores / 32 threads) +- Container runtime used by benchmark: `Docker 28.3.3` + +Identical hardware to all prior benchmark runs. diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.0.json b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.0.json new file mode 100644 index 000000000..267ebc201 --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.0.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "74f5c8a9305912db8873024156cc006662ad1902", + "driver": "mysql", + "db_version": "8.0", + "ops": 100, + "timestamp": "2026-05-01T09:58:41.161303801+00:00", + "timings_ms": { + "benchmark": 7270, + "report_build": 1, + "total": 7272 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 737, + "median_us": 984, + "worst_us": 1537 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 103, + "median_us": 115, + "worst_us": 290 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 161, + "median_us": 171, + "worst_us": 343 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 895, + "median_us": 1005, + "worst_us": 1897 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 952, + "median_us": 1066, + "worst_us": 1495 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 106, + "median_us": 137, + "worst_us": 301 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 924, + "median_us": 1036, + "worst_us": 2144 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 731, + "median_us": 981, + "worst_us": 2852 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 100, + "median_us": 118, + "worst_us": 281 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 160, + "median_us": 175, + "worst_us": 299 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 719, + "median_us": 962, + "worst_us": 3573 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 754, + "median_us": 974, + "worst_us": 1394 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 103, + "median_us": 129, + "worst_us": 319 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 166, + "median_us": 189, + "worst_us": 371 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 796, + "median_us": 994, + "worst_us": 1825 + } + ] +} diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.4.json b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.4.json new file mode 100644 index 000000000..ffe1288c5 --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/mysql-8.4.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "74f5c8a9305912db8873024156cc006662ad1902", + "driver": "mysql", + "db_version": "8.4", + "ops": 100, + "timestamp": "2026-05-01T09:58:23.545474317+00:00", + "timings_ms": { + "benchmark": 6371, + "report_build": 1, + "total": 6372 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 692, + "median_us": 769, + "worst_us": 1878 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 95, + "median_us": 112, + "worst_us": 266 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 152, + "median_us": 172, + "worst_us": 429 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 711, + "median_us": 773, + "worst_us": 1333 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 708, + "median_us": 793, + "worst_us": 1301 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 94, + "median_us": 115, + "worst_us": 258 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 706, + "median_us": 774, + "worst_us": 1811 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 685, + "median_us": 735, + "worst_us": 1156 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 102, + "median_us": 109, + "worst_us": 266 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 143, + "median_us": 161, + "worst_us": 262 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 681, + "median_us": 766, + "worst_us": 1549 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 687, + "median_us": 750, + "worst_us": 1201 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 95, + "median_us": 118, + "worst_us": 336 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 156, + "median_us": 167, + "worst_us": 289 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 686, + "median_us": 739, + "worst_us": 1175 + } + ] +} diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-05-01/postgresql-17.json b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/postgresql-17.json new file mode 100644 index 000000000..e24aa18ac --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/postgresql-17.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "74f5c8a9305912db8873024156cc006662ad1902", + "driver": "postgresql", + "db_version": "17", + "ops": 100, + "timestamp": "2026-05-01T09:56:57.467226419+00:00", + "timings_ms": { + "benchmark": 1450, + "report_build": 1, + "total": 1451 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 269, + "median_us": 298, + "worst_us": 652 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 81, + "median_us": 88, + "worst_us": 539 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 137, + "median_us": 146, + "worst_us": 290 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 266, + "median_us": 302, + "worst_us": 500 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 266, + "median_us": 299, + "worst_us": 648 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 82, + "median_us": 86, + "worst_us": 401 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 275, + "median_us": 305, + "worst_us": 829 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 270, + "median_us": 294, + "worst_us": 632 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 82, + "median_us": 95, + "worst_us": 285 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 123, + "median_us": 135, + "worst_us": 247 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 267, + "median_us": 293, + "worst_us": 426 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 265, + "median_us": 292, + "worst_us": 567 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 81, + "median_us": 95, + "worst_us": 290 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 137, + "median_us": 155, + "worst_us": 228 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 265, + "median_us": 300, + "worst_us": 537 + } + ] +} diff --git a/packages/tracker-core/docs/benchmarking/runs/2026-05-01/sqlite3.json b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/sqlite3.json new file mode 100644 index 000000000..be53f746b --- /dev/null +++ b/packages/tracker-core/docs/benchmarking/runs/2026-05-01/sqlite3.json @@ -0,0 +1,121 @@ +{ + "meta": { + "git_revision": "74f5c8a9305912db8873024156cc006662ad1902", + "driver": "sqlite3", + "db_version": "-", + "ops": 100, + "timestamp": "2026-05-01T09:57:47.730740066+00:00", + "timings_ms": { + "benchmark": 117, + "report_build": 1, + "total": 119 + } + }, + "operations": [ + { + "name": "save_torrent_downloads", + "count": 100, + "best_us": 77, + "median_us": 89, + "worst_us": 185 + }, + { + "name": "load_torrent_downloads", + "count": 100, + "best_us": 21, + "median_us": 23, + "worst_us": 62 + }, + { + "name": "load_all_torrents_downloads", + "count": 100, + "best_us": 70, + "median_us": 77, + "worst_us": 116 + }, + { + "name": "increase_downloads_for_torrent", + "count": 100, + "best_us": 66, + "median_us": 70, + "worst_us": 108 + }, + { + "name": "save_global_downloads", + "count": 100, + "best_us": 74, + "median_us": 76, + "worst_us": 161 + }, + { + "name": "load_global_downloads", + "count": 100, + "best_us": 21, + "median_us": 21, + "worst_us": 40 + }, + { + "name": "increase_global_downloads", + "count": 100, + "best_us": 65, + "median_us": 67, + "worst_us": 142 + }, + { + "name": "add_info_hash_to_whitelist", + "count": 100, + "best_us": 77, + "median_us": 81, + "worst_us": 166 + }, + { + "name": "get_info_hash_from_whitelist", + "count": 100, + "best_us": 21, + "median_us": 21, + "worst_us": 105 + }, + { + "name": "load_whitelist", + "count": 100, + "best_us": 51, + "median_us": 55, + "worst_us": 73 + }, + { + "name": "remove_info_hash_from_whitelist", + "count": 100, + "best_us": 71, + "median_us": 81, + "worst_us": 154 + }, + { + "name": "add_key_to_keys", + "count": 100, + "best_us": 79, + "median_us": 81, + "worst_us": 142 + }, + { + "name": "get_key_from_keys", + "count": 100, + "best_us": 21, + "median_us": 22, + "worst_us": 44 + }, + { + "name": "load_keys", + "count": 100, + "best_us": 72, + "median_us": 77, + "worst_us": 129 + }, + { + "name": "remove_key_from_keys", + "count": 100, + "best_us": 70, + "median_us": 73, + "worst_us": 116 + } + ] +} From 248df3d9e1d08ba28f9d5b570f5c8f5ced1c46d4 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:29:46 +0100 Subject: [PATCH 09/22] ci(container): isolate compose build paths from storage --- .github/workflows/container.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml index 7e8ffa442..fa8c2a855 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yaml @@ -50,7 +50,32 @@ jobs: - id: compose name: Compose - run: docker compose build + run: | + QBT_E2E_WORKDIR="${RUNNER_TEMP}/qbt-e2e-compose-build" + mkdir -p "${QBT_E2E_WORKDIR}/tracker-storage" + mkdir -p "${QBT_E2E_WORKDIR}/seeder-config" + mkdir -p "${QBT_E2E_WORKDIR}/seeder-downloads" + mkdir -p "${QBT_E2E_WORKDIR}/leecher-config" + mkdir -p "${QBT_E2E_WORKDIR}/leecher-downloads" + mkdir -p "${QBT_E2E_WORKDIR}/shared" + + export QBT_E2E_TRACKER_IMAGE=torrust-tracker:local + export QBT_E2E_QBITTORRENT_IMAGE=ghcr.io/linuxserver/qbittorrent:latest + export QBT_E2E_TRACKER_CONFIG_PATH=./share/default/config/tracker.container.sqlite3.toml + export QBT_E2E_TRACKER_STORAGE_PATH="${QBT_E2E_WORKDIR}/tracker-storage" + export QBT_E2E_TRACKER_HTTP_TRACKER_PORT=7070 + export QBT_E2E_TRACKER_UDP_PORT=6969 + export QBT_E2E_TRACKER_HTTP_API_PORT=1212 + export QBT_E2E_TRACKER_HEALTH_CHECK_API_PORT=1313 + export QBT_E2E_SEEDER_CONFIG_PATH="${QBT_E2E_WORKDIR}/seeder-config" + export QBT_E2E_SEEDER_DOWNLOADS_PATH="${QBT_E2E_WORKDIR}/seeder-downloads" + export QBT_E2E_LEECHER_CONFIG_PATH="${QBT_E2E_WORKDIR}/leecher-config" + export QBT_E2E_LEECHER_DOWNLOADS_PATH="${QBT_E2E_WORKDIR}/leecher-downloads" + export QBT_E2E_SHARED_PATH="${QBT_E2E_WORKDIR}/shared" + + docker compose -f compose.qbittorrent-e2e.sqlite3.yaml build + docker compose -f compose.qbittorrent-e2e.mysql.yaml build + docker compose -f compose.qbittorrent-e2e.postgresql.yaml build context: name: Context From b0a654ee9fd7210e7d76ec64cdfd57c612b7e84f Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:31:02 +0100 Subject: [PATCH 10/22] chore(repo): remove legacy compose file references --- AGENTS.md | 28 +++++++++++++++------------- compose.yaml | 51 --------------------------------------------------- 2 files changed, 15 insertions(+), 64 deletions(-) delete mode 100644 compose.yaml diff --git a/AGENTS.md b/AGENTS.md index cda2ae240..a1da818e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,19 +106,21 @@ All packages live under `packages/`. The workspace version is `3.0.0-develop`. ## 📄 Key Configuration Files -| File | Used by | -| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `.markdownlint.json` | markdownlint | -| `.yamllint-ci.yml` | yamllint | -| `.taplo.toml` | taplo (TOML formatting) | -| `cspell.json` | cspell (spell checker) configuration | -| `project-words.txt` | cspell project-specific dictionary | -| `rustfmt.toml` | rustfmt (`group_imports = "StdExternalCrate"`, `max_width = 130`) | -| `.cargo/config.toml` | Cargo aliases (`cov`, `cov-lcov`, `cov-html`, `time`) and global `rustflags` (`-D warnings`, `-D unused`, `-D rust-2018-idioms`, …) | -| `Cargo.toml` | Cargo workspace root | -| `compose.yaml` | Docker Compose for local dev and demo | -| `Containerfile` | Container image definition | -| `codecov.yaml` | Code coverage configuration | +| File | Used by | +| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `.markdownlint.json` | markdownlint | +| `.yamllint-ci.yml` | yamllint | +| `.taplo.toml` | taplo (TOML formatting) | +| `cspell.json` | cspell (spell checker) configuration | +| `project-words.txt` | cspell project-specific dictionary | +| `rustfmt.toml` | rustfmt (`group_imports = "StdExternalCrate"`, `max_width = 130`) | +| `.cargo/config.toml` | Cargo aliases (`cov`, `cov-lcov`, `cov-html`, `time`) and global `rustflags` (`-D warnings`, `-D unused`, `-D rust-2018-idioms`, …) | +| `Cargo.toml` | Cargo workspace root | +| `compose.qbittorrent-e2e.sqlite3.yaml` | qBittorrent E2E Compose stack for SQLite backend | +| `compose.qbittorrent-e2e.mysql.yaml` | qBittorrent E2E Compose stack for MySQL backend | +| `compose.qbittorrent-e2e.postgresql.yaml` | qBittorrent E2E Compose stack for PostgreSQL backend | +| `Containerfile` | Container image definition | +| `codecov.yaml` | Code coverage configuration | ## 🧪 Build & Test diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index c2e7c63bd..000000000 --- a/compose.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: torrust -services: - tracker: - image: torrust-tracker:release - tty: true - environment: - - TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=${TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER:-mysql} - - TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN=${TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN:-MyAccessToken} - networks: - - server_side - ports: - - 6969:6969/udp - - 7070:7070 - - 1212:1212 - volumes: - - ./storage/tracker/lib:/var/lib/torrust/tracker:Z - - ./storage/tracker/log:/var/log/torrust/tracker:Z - - ./storage/tracker/etc:/etc/torrust/tracker:Z - depends_on: - - mysql - - mysql: - image: mysql:8.0 - command: "--default-authentication-plugin=mysql_native_password" - healthcheck: - test: - [ - "CMD-SHELL", - 'mysqladmin ping -h 127.0.0.1 --password="$$(cat /run/secrets/db-password)" --silent', - ] - interval: 3s - retries: 5 - start_period: 30s - environment: - - MYSQL_ROOT_HOST=% - - MYSQL_ROOT_PASSWORD=root_secret_password - - MYSQL_DATABASE=torrust_tracker - - MYSQL_USER=db_user - - MYSQL_PASSWORD=db_user_secret_password - networks: - - server_side - ports: - - 3306:3306 - volumes: - - mysql_data:/var/lib/mysql - -networks: - server_side: {} - -volumes: - mysql_data: {} From 3ef070714429562c2496f4292ba0164341e1f510 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:31:50 +0100 Subject: [PATCH 11/22] docs(containers): document PostgreSQL runtime configuration --- README.md | 4 ++-- docs/containers.md | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2fe28db08..0f0dd984f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ - [x] Private & Whitelisted mode. - [x] Tracker Management API. - [x] Support [newTrackon][newtrackon] checks. -- [x] Persistent `SQLite3` or `MySQL` Databases. +- [x] Persistent `SQLite3`, `MySQL`, or `PostgreSQL` Databases. ## Tracker Demo @@ -46,7 +46,7 @@ Core: Persistence: -- [ ] Support other databases like PostgreSQL. +- [ ] Support additional persistence backends. Performance: diff --git a/docs/containers.md b/docs/containers.md index a7754d8aa..4d797ce83 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -149,7 +149,7 @@ The following environmental variables can be set: - `TORRUST_TRACKER_CONFIG_TOML_PATH` - The in-container path to the tracker configuration file, (default: `"/etc/torrust/tracker/tracker.toml"`). - `TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN` - Override of the admin token. If set, this value overrides any value set in the config. -- `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER` - The database type used for the container, (options: `sqlite3`, `mysql`, default `sqlite3`). Please Note: This dose not override the database configuration within the `.toml` config file. +- `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER` - The database type used for the container, (options: `sqlite3`, `mysql`, `postgresql`, default `sqlite3`). Please Note: This dose not override the database configuration within the `.toml` config file. - `TORRUST_TRACKER_CONFIG_TOML` - Load config from this environmental variable instead from a file, (i.e: `TORRUST_TRACKER_CONFIG_TOML=$(cat tracker-tracker.toml)`). - `USER_ID` - The user id for the runtime crated `torrust` user. Please Note: This user id should match the ownership of the host-mapped volumes, (default `1000`). - `UDP_PORT` - The port for the UDP tracker. This should match the port used in the configuration, (default `6969`). @@ -157,6 +157,19 @@ The following environmental variables can be set: - `API_PORT` - The port for the tracker API. This should match the port used in the configuration, (default `1212`). - `HEALTH_CHECK_API_PORT` - The port for the Health Check API. This should match the port used in the configuration, (default `1313`). +#### PostgreSQL backend notes + +To run the tracker with PostgreSQL in containers: + +- Set `TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=postgresql`. +- Use the default PostgreSQL container configuration file: + `share/default/config/tracker.container.postgresql.toml`. +- Ensure the target database exists before tracker startup. + The default PostgreSQL DSN in the container config expects `torrust_tracker`. + +When using a PostgreSQL container, set `POSTGRES_DB=torrust_tracker` (or create the +same database explicitly) so the tracker can connect at startup. + ### Sockets Socket ports used internally within the container can be mapped to with the `--publish` argument. From 3c9264cf6f5125cb6134078edaa28b19dc926bc7 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:32:14 +0100 Subject: [PATCH 12/22] docs(issues): mark task 9 complete in 1723 spec --- .../1723-1525-08-add-postgresql-driver.md | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/issues/1723-1525-08-add-postgresql-driver.md b/docs/issues/1723-1525-08-add-postgresql-driver.md index deafd990b..4ff0b690d 100644 --- a/docs/issues/1723-1525-08-add-postgresql-driver.md +++ b/docs/issues/1723-1525-08-add-postgresql-driver.md @@ -789,7 +789,7 @@ Acceptance criteria: lists all three supported backends. - [ ] `compose.yaml` is renamed to `compose.mysql.yaml`; `compose.postgresql.yaml` exists; both are validated by `.github/workflows/container.yaml`; `docker compose -f - compose.postgresql.yaml up` starts the tracker successfully against PostgreSQL. +compose.postgresql.yaml up` starts the tracker successfully against PostgreSQL. - [ ] `project-words.txt` is up to date; `linter cspell` reports no failures. - [ ] `README.md` lists PostgreSQL as a supported database backend. - [ ] `docs/containers.md` documents how to run the tracker with PostgreSQL and states the @@ -934,6 +934,43 @@ After all commits, run benchmarks and update baseline artifacts in a final commi (all changes are scoped). Phase 2 tasks depend only on Phase 1 being complete. Benchmarks (Phase 3) run last for data freshness. +## Progress Update (2026-05-01) + +Status by task (based on commits currently on this branch): + +- [x] Task 1: configuration `Driver::PostgreSQL` + URL secret masking. +- [x] Task 2: `sqlx` postgres feature + PostgreSQL migration set. +- [x] Task 3: PostgreSQL driver implementation. +- [x] Task 4: factory/setup wiring for PostgreSQL. +- [x] Task 5: PostgreSQL driver tests. +- [x] Task 6: compatibility matrix extended with PostgreSQL versions. +- [x] Task 7: qBittorrent E2E runner extended for MySQL/PostgreSQL. +- [x] Task 8: benchmark runner extended with PostgreSQL and first benchmark run committed. +- [x] Task 9: container compose strategy and user-facing container docs updates. + +Recent milestone commits: + +- `a0f9c001` — PostgreSQL database driver. +- `15af1e07` — PostgreSQL key timestamp fix. +- `54210f3f` — PostgreSQL compatibility job. +- `74f5c8a9` — qBittorrent E2E runner MySQL/PostgreSQL extension. +- `e0d0a872` — benchmark runner PostgreSQL startup/wait fix. +- `aee2efbe` — benchmark artifacts and report for `2026-05-01`. +- `248df3d9` — container compose validation uses isolated temp paths. +- `b0a654ee` — legacy `compose.yaml` removed and compose references aligned. +- `3ef07071` — README and containers guide updated for PostgreSQL runtime usage. + +Scope note for Task 8: + +- The benchmark integration in this branch uses the Rust benchmark runner in + `packages/tracker-core` with containerized DB lifecycle managed from the runner/test harness, + and stores artifacts under `packages/tracker-core/docs/benchmarking/`. + +Task 9 implementation note: + +- The container validation workflow now uses the qBittorrent E2E compose files and isolated + temporary paths, instead of the legacy root `compose.yaml` stack. + ## References - EPIC: `#1525` — `docs/issues/1525-overhaul-persistence.md` From 517b42e3cb2d1f470cac6e159f87429e9c291fba Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:51:32 +0100 Subject: [PATCH 13/22] ci(db-compatibility): extract database compatibility workflow Move database-compatibility-mysql and database-compatibility-postgres jobs from testing.yaml into a dedicated db-compatibility.yaml workflow. Motivation: the compat jobs only exercise the tracker-core package. Keeping them in a separate file makes the workflow portable if tracker-core is ever extracted to its own repository, and reduces the size of testing.yaml. The e2e job in testing.yaml now depends only on unit (the cross-workflow gate via database-compatibility was a soft ordering preference, not a hard requirement). --- .github/workflows/db-compatibility.yaml | 69 +++++++++++++++++++++++++ .github/workflows/testing.yaml | 64 +---------------------- 2 files changed, 70 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/db-compatibility.yaml diff --git a/.github/workflows/db-compatibility.yaml b/.github/workflows/db-compatibility.yaml new file mode 100644 index 000000000..abbdf2c7d --- /dev/null +++ b/.github/workflows/db-compatibility.yaml @@ -0,0 +1,69 @@ +name: Database Compatibility + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + database-compatibility-mysql: + name: Database Compatibility MySQL (${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + + strategy: + matrix: + mysql-version: ["8.0", "8.4"] + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: database + name: Run Database Compatibility Test + env: + TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST: "true" + TORRUST_TRACKER_CORE_MYSQL_DRIVER_IMAGE_TAG: ${{ matrix.mysql-version }} + run: cargo test -p bittorrent-tracker-core --features db-compatibility-tests run_mysql_driver_tests -- --nocapture + + database-compatibility-postgres: + name: Database Compatibility PostgreSQL (${{ matrix.postgres-version }}) + runs-on: ubuntu-latest + + strategy: + matrix: + postgres-version: ["15", "16", "17"] + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: database + name: Run Database Compatibility Test + env: + TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST: "true" + TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG: ${{ matrix.postgres-version }} + run: cargo test -p bittorrent-tracker-core --features db-compatibility-tests run_postgres_driver_tests -- --nocapture diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index b07d6267a..e0b2731ed 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -133,72 +133,10 @@ jobs: name: Run Unit Tests run: cargo test --tests --benches --examples --workspace --all-targets --all-features - database-compatibility-mysql: - name: Database Compatibility MySQL (${{ matrix.mysql-version }}) - runs-on: ubuntu-latest - needs: unit - - strategy: - matrix: - mysql-version: ["8.0", "8.4"] - - steps: - - id: checkout - name: Checkout Repository - uses: actions/checkout@v6 - - - id: setup - name: Setup Toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - - - id: cache - name: Enable Job Cache - uses: Swatinem/rust-cache@v2 - - - id: database - name: Run Database Compatibility Test - env: - TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST: "true" - TORRUST_TRACKER_CORE_MYSQL_DRIVER_IMAGE_TAG: ${{ matrix.mysql-version }} - run: cargo test -p bittorrent-tracker-core --features db-compatibility-tests run_mysql_driver_tests -- --nocapture - - database-compatibility-postgres: - name: Database Compatibility PostgreSQL (${{ matrix.postgres-version }}) - runs-on: ubuntu-latest - needs: unit - - strategy: - matrix: - postgres-version: ["15", "16", "17"] - - steps: - - id: checkout - name: Checkout Repository - uses: actions/checkout@v6 - - - id: setup - name: Setup Toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - - - id: cache - name: Enable Job Cache - uses: Swatinem/rust-cache@v2 - - - id: database - name: Run Database Compatibility Test - env: - TORRUST_TRACKER_CORE_RUN_POSTGRES_DRIVER_TEST: "true" - TORRUST_TRACKER_CORE_POSTGRES_DRIVER_IMAGE_TAG: ${{ matrix.postgres-version }} - run: cargo test -p bittorrent-tracker-core --features db-compatibility-tests run_postgres_driver_tests -- --nocapture - e2e: name: E2E runs-on: ubuntu-latest - needs: [database-compatibility-mysql, database-compatibility-postgres] + needs: unit timeout-minutes: 45 strategy: From 8fb55e943c5f2e1b178641640a509cf7ccf23100 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:52:12 +0100 Subject: [PATCH 14/22] ci(db-benchmarking): add persistence benchmark smoke workflow Add a new db-benchmarking.yaml workflow that runs the persistence_benchmark_runner binary against all three drivers on every push and pull request. Each job uses --ops 10 to keep runtime short while still exercising the full binary path: argument parsing, testcontainers startup, schema creation, query execution, and JSON report emission. A non-zero exit code (panic, missing container, broken SQL) fails the job immediately, catching runtime regressions that compilation alone cannot detect. --- .github/workflows/db-benchmarking.yaml | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/db-benchmarking.yaml diff --git a/.github/workflows/db-benchmarking.yaml b/.github/workflows/db-benchmarking.yaml new file mode 100644 index 000000000..b73014aae --- /dev/null +++ b/.github/workflows/db-benchmarking.yaml @@ -0,0 +1,78 @@ +name: Database Benchmarking + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + persistence-benchmark-sqlite3: + name: Persistence Benchmark SQLite3 + runs-on: ubuntu-latest + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: benchmark + name: Run Persistence Benchmark (SQLite3) + run: cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver sqlite3 --ops 10 + + persistence-benchmark-mysql: + name: Persistence Benchmark MySQL + runs-on: ubuntu-latest + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: benchmark + name: Run Persistence Benchmark (MySQL) + run: cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver mysql --db-version 8.4 --ops 10 + + persistence-benchmark-postgresql: + name: Persistence Benchmark PostgreSQL + runs-on: ubuntu-latest + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: benchmark + name: Run Persistence Benchmark (PostgreSQL) + run: cargo run -p bittorrent-tracker-core --bin persistence_benchmark_runner -- --driver postgresql --db-version 17 --ops 10 From 87d458c499350e0f69265caec2fceac5fc264054 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:52:44 +0100 Subject: [PATCH 15/22] docs(readme): add workflow badges for db-compatibility and db-benchmarking --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f0dd984f..6fc746418 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Torrust Tracker -[![container_wf_b]][container_wf] [![coverage_wf_b]][coverage_wf] [![deployment_wf_b]][deployment_wf] [![testing_wf_b]][testing_wf] +[![container_wf_b]][container_wf] [![coverage_wf_b]][coverage_wf] [![deployment_wf_b]][deployment_wf] [![testing_wf_b]][testing_wf] [![db_compat_wf_b]][db_compat_wf] [![db_bench_wf_b]][db_bench_wf] **Torrust Tracker** is a [BitTorrent][bittorrent] Tracker that matchmakes peers and collects statistics. Written in [Rust Language][rust] with the [Axum] web framework. **This tracker aims to be respectful to established standards, (both [formal][BEP 00] and [otherwise][torrent_source_felid]).** @@ -250,6 +250,10 @@ This project was a joint effort by [Nautilus Cyberneering GmbH][nautilus] and [D [deployment_wf_b]: ../../actions/workflows/deployment.yaml/badge.svg [testing_wf]: ../../actions/workflows/testing.yaml [testing_wf_b]: ../../actions/workflows/testing.yaml/badge.svg +[db_compat_wf]: ../../actions/workflows/db-compatibility.yaml +[db_compat_wf_b]: ../../actions/workflows/db-compatibility.yaml/badge.svg +[db_bench_wf]: ../../actions/workflows/db-benchmarking.yaml +[db_bench_wf_b]: ../../actions/workflows/db-benchmarking.yaml/badge.svg [bittorrent]: http://bittorrent.org/ [rust]: https://www.rust-lang.org/ [axum]: https://github.com/tokio-rs/axum From 661bbd93125667b97f7fe345661e41e1a4514eb7 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 11:54:18 +0100 Subject: [PATCH 16/22] ci(os-compatibility): extract build matrix into dedicated workflow Move the cross-platform build job from testing.yaml into a new os-compatibility.yaml workflow. The build job has no dependency on any other job and is solely concerned with compilation portability across Ubuntu, macOS, and Windows. Keeping it separate follows the same principle applied to db-compatibility and db-benchmarking: one workflow, one concern. Also adds the os-compatibility badge to README.md. --- .github/workflows/os-compatibility.yaml | 31 +++++++++++++++++++++++++ .github/workflows/testing.yaml | 22 ------------------ README.md | 4 +++- 3 files changed, 34 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/os-compatibility.yaml diff --git a/.github/workflows/os-compatibility.yaml b/.github/workflows/os-compatibility.yaml new file mode 100644 index 000000000..9f9b818c6 --- /dev/null +++ b/.github/workflows/os-compatibility.yaml @@ -0,0 +1,31 @@ +name: OS Compatibility + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build on ${{ matrix.os }} (${{ matrix.toolchain }}) + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + toolchain: [nightly, stable] + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.toolchain }} + + - name: Build project + run: cargo build --verbose diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index e0b2731ed..29a9f33ad 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -72,28 +72,6 @@ jobs: name: Run All Linters run: linter all - build: - name: Build on ${{ matrix.os }} (${{ matrix.toolchain }}) - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - toolchain: [nightly, stable] - - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - id: setup - name: Setup Toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ matrix.toolchain }} - - - name: Build project - run: cargo build --verbose - unit: name: Units runs-on: ubuntu-latest diff --git a/README.md b/README.md index 6fc746418..b18927e09 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Torrust Tracker -[![container_wf_b]][container_wf] [![coverage_wf_b]][coverage_wf] [![deployment_wf_b]][deployment_wf] [![testing_wf_b]][testing_wf] [![db_compat_wf_b]][db_compat_wf] [![db_bench_wf_b]][db_bench_wf] +[![container_wf_b]][container_wf] [![coverage_wf_b]][coverage_wf] [![deployment_wf_b]][deployment_wf] [![testing_wf_b]][testing_wf] [![os_compat_wf_b]][os_compat_wf] [![db_compat_wf_b]][db_compat_wf] [![db_bench_wf_b]][db_bench_wf] **Torrust Tracker** is a [BitTorrent][bittorrent] Tracker that matchmakes peers and collects statistics. Written in [Rust Language][rust] with the [Axum] web framework. **This tracker aims to be respectful to established standards, (both [formal][BEP 00] and [otherwise][torrent_source_felid]).** @@ -250,6 +250,8 @@ This project was a joint effort by [Nautilus Cyberneering GmbH][nautilus] and [D [deployment_wf_b]: ../../actions/workflows/deployment.yaml/badge.svg [testing_wf]: ../../actions/workflows/testing.yaml [testing_wf_b]: ../../actions/workflows/testing.yaml/badge.svg +[os_compat_wf]: ../../actions/workflows/os-compatibility.yaml +[os_compat_wf_b]: ../../actions/workflows/os-compatibility.yaml/badge.svg [db_compat_wf]: ../../actions/workflows/db-compatibility.yaml [db_compat_wf_b]: ../../actions/workflows/db-compatibility.yaml/badge.svg [db_bench_wf]: ../../actions/workflows/db-benchmarking.yaml From 33e58200d035c0b6c832ec20fe2cbb27b25968a8 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 12:03:16 +0100 Subject: [PATCH 17/22] ci(testing): collapse matrix jobs into test-nightly and test-stable Replace the four-job chain (format -> check -> unit -> e2e) with two flat jobs: test-nightly and test-stable. Each job runs checkout, toolchain setup, Node.js, Rust cache, dependency fetch, linter install, and tool install once, then executes all stages sequentially. Motivation: the previous structure recompiled the entire workspace multiple times per push across format, check, unit, and e2e jobs, each starting from a cold runner. Collapsing into two jobs eliminates the redundant compilations while keeping nightly and stable coverage separate. A cargo fetch step is added after cache restore in both jobs to make dependency download time visible in the job timeline. Differences from the old structure: - fmt check runs only under nightly (rustfmt nightly is the canonical formatter for this repo) - qBittorrent E2E tests (all three backends) run only under stable, matching the previous if: matrix.toolchain == 'stable' guard - test-nightly timeout: 45 min; test-stable timeout: 90 min --- .github/workflows/testing.yaml | 128 ++++++++++++++------------------- 1 file changed, 55 insertions(+), 73 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 29a9f33ad..a30998b6a 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -8,9 +8,10 @@ env: CARGO_TERM_COLOR: always jobs: - format: - name: Formatting + test-nightly: + name: Test (nightly) runs-on: ubuntu-latest + timeout-minutes: 45 steps: - id: checkout @@ -22,37 +23,7 @@ jobs: uses: dtolnay/rust-toolchain@stable with: toolchain: nightly - components: rustfmt - - - id: cache - name: Enable Workflow Cache - uses: Swatinem/rust-cache@v2 - - - id: format - name: Run Formatting-Checks - run: cargo fmt --check - - check: - name: Linting - runs-on: ubuntu-latest - needs: format - timeout-minutes: 15 - - strategy: - matrix: - toolchain: [nightly, stable] - - steps: - - id: checkout - name: Checkout Repository - uses: actions/checkout@v6 - - - id: setup - name: Setup Toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ matrix.toolchain }} - components: clippy, rustfmt + components: rustfmt, clippy, llvm-tools-preview - id: node name: Setup Node.js @@ -61,25 +32,47 @@ jobs: node-version: "20" - id: cache - name: Enable Workflow Cache + name: Enable Job Cache uses: Swatinem/rust-cache@v2 - - id: tools + - id: fetch + name: Download Dependencies + run: cargo fetch --verbose + + - id: linter name: Install Internal Linter run: cargo install --locked --git https://github.com/torrust/torrust-linting --bin linter + - id: tools + name: Install Tools + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov, cargo-nextest + + - id: format + name: Run Formatting-Checks + run: cargo fmt --check + - id: lint name: Run All Linters run: linter all - unit: - name: Units - runs-on: ubuntu-latest - needs: check + - id: test-docs + name: Run Documentation Tests + run: cargo test --doc --workspace - strategy: - matrix: - toolchain: [nightly, stable] + - id: test + name: Run Unit Tests + run: cargo test --tests --benches --examples --workspace --all-targets --all-features + + - id: run-tracker-e2e-tests + name: Run E2E Tests + run: cargo run --bin e2e_tests_runner -- --config-toml-path "./share/default/config/tracker.e2e.container.sqlite3.toml" + + test-stable: + name: Test (stable) + runs-on: ubuntu-latest + timeout-minutes: 90 steps: - id: checkout @@ -90,19 +83,37 @@ jobs: name: Setup Toolchain uses: dtolnay/rust-toolchain@stable with: - toolchain: ${{ matrix.toolchain }} - components: llvm-tools-preview + toolchain: stable + components: clippy, llvm-tools-preview + + - id: node + name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "20" - id: cache name: Enable Job Cache uses: Swatinem/rust-cache@v2 + - id: fetch + name: Download Dependencies + run: cargo fetch --verbose + + - id: linter + name: Install Internal Linter + run: cargo install --locked --git https://github.com/torrust/torrust-linting --bin linter + - id: tools name: Install Tools uses: taiki-e/install-action@v2 with: tool: cargo-llvm-cov, cargo-nextest + - id: lint + name: Run All Linters + run: linter all + - id: test-docs name: Run Documentation Tests run: cargo test --doc --workspace @@ -111,47 +122,18 @@ jobs: name: Run Unit Tests run: cargo test --tests --benches --examples --workspace --all-targets --all-features - e2e: - name: E2E - runs-on: ubuntu-latest - needs: unit - timeout-minutes: 45 - - strategy: - matrix: - toolchain: [nightly, stable] - - steps: - - id: setup-e2e-toolchain - name: Setup Toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ matrix.toolchain }} - components: llvm-tools-preview - - - id: enable-e2e-job-cache - name: Enable Job Cache - uses: Swatinem/rust-cache@v2 - - - id: checkout-repository - name: Checkout Repository - uses: actions/checkout@v6 - - id: run-tracker-e2e-tests name: Run E2E Tests run: cargo run --bin e2e_tests_runner -- --config-toml-path "./share/default/config/tracker.e2e.container.sqlite3.toml" - id: run-qbittorrent-e2e-test - if: matrix.toolchain == 'stable' name: Run qBittorrent E2E Test (SQLite) run: cargo run --bin qbittorrent_e2e_runner -- --compose-file ./compose.qbittorrent-e2e.sqlite3.yaml --timeout-seconds 600 - id: run-qbittorrent-e2e-test-mysql - if: matrix.toolchain == 'stable' name: Run qBittorrent E2E Test (MySQL) run: cargo run --bin qbittorrent_e2e_runner -- --db-driver mysql --timeout-seconds 600 - id: run-qbittorrent-e2e-test-postgresql - if: matrix.toolchain == 'stable' name: Run qBittorrent E2E Test (PostgreSQL) run: cargo run --bin qbittorrent_e2e_runner -- --db-driver postgresql --timeout-seconds 600 From 40ed6696f398d00adc96a32c40b78763f19461f4 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 12:08:58 +0100 Subject: [PATCH 18/22] chore(issues): archive closed issue specs to docs/issues/closed Move 10 closed issue spec files from docs/issues/ to the new docs/issues/closed/ buffer folder: #523 internal-linting-tool #1697 ai-agent-configuration #1703 1525-01-persistence-test-coverage #1706 1525-02-qbittorrent-e2e #1710 1525-03-persistence-benchmarking #1713 1525-04-split-persistence-traits #1715 1525-04b-migrate-consumers-to-narrow-traits #1717 1525-05-migrate-sqlite-and-mysql-to-sqlx #1719 1525-06-introduce-schema-migrations #1721 1525-07-align-rust-and-db-types Add docs/issues/closed/README.md explaining the two-stage lifecycle (archive then delete) for closed spec files. Update the cleanup-completed-issues skill (v1.1) to reflect the new process: move to closed/ first, delete permanently only when no longer referenced by active work. --- .../cleanup-completed-issues/SKILL.md | 75 ++++++++++--------- .../1697-ai-agent-configuration.md | 0 .../1703-1525-01-persistence-test-coverage.md | 0 .../1706-1525-02-qbittorrent-e2e.md | 0 .../1710-1525-03-persistence-benchmarking.md | 0 .../1713-1525-04-split-persistence-traits.md | 0 ...-04b-migrate-consumers-to-narrow-traits.md | 0 ...525-05-migrate-sqlite-and-mysql-to-sqlx.md | 0 ...719-1525-06-introduce-schema-migrations.md | 0 .../1721-1525-07-align-rust-and-db-types.md | 0 .../{ => closed}/523-internal-linting-tool.md | 0 docs/issues/closed/README.md | 23 ++++++ 12 files changed, 63 insertions(+), 35 deletions(-) rename docs/issues/{ => closed}/1697-ai-agent-configuration.md (100%) rename docs/issues/{ => closed}/1703-1525-01-persistence-test-coverage.md (100%) rename docs/issues/{ => closed}/1706-1525-02-qbittorrent-e2e.md (100%) rename docs/issues/{ => closed}/1710-1525-03-persistence-benchmarking.md (100%) rename docs/issues/{ => closed}/1713-1525-04-split-persistence-traits.md (100%) rename docs/issues/{ => closed}/1715-1525-04b-migrate-consumers-to-narrow-traits.md (100%) rename docs/issues/{ => closed}/1717-1525-05-migrate-sqlite-and-mysql-to-sqlx.md (100%) rename docs/issues/{ => closed}/1719-1525-06-introduce-schema-migrations.md (100%) rename docs/issues/{ => closed}/1721-1525-07-align-rust-and-db-types.md (100%) rename docs/issues/{ => closed}/523-internal-linting-tool.md (100%) create mode 100644 docs/issues/closed/README.md diff --git a/.github/skills/dev/planning/cleanup-completed-issues/SKILL.md b/.github/skills/dev/planning/cleanup-completed-issues/SKILL.md index a4c7b3966..5d3ef19c8 100644 --- a/.github/skills/dev/planning/cleanup-completed-issues/SKILL.md +++ b/.github/skills/dev/planning/cleanup-completed-issues/SKILL.md @@ -1,33 +1,36 @@ --- name: cleanup-completed-issues -description: Guide for cleaning up completed and closed issues in the torrust-tracker project. Covers removing issue documentation files from docs/issues/ and committing the cleanup. Supports single issue cleanup or batch cleanup. Use when cleaning up closed issues, removing issue docs, or maintaining the docs/issues/ folder. Triggers on "cleanup issue", "remove issue", "clean completed issues", "delete closed issue", or "maintain issue docs". +description: Guide for cleaning up completed and closed issues in the torrust-tracker project. Covers moving closed issue documentation files from docs/issues/ to docs/issues/closed/ and eventually deleting them. Supports single issue cleanup or batch cleanup. Use when cleaning up closed issues, archiving issue docs, or maintaining the docs/issues/ folder. Triggers on "cleanup issue", "archive issue", "move closed issue", "clean completed issues", "delete closed issue", or "maintain issue docs". metadata: author: torrust - version: "1.0" + version: "1.1" --- # Cleaning Up Completed Issues -## When to Clean Up +## Two-Stage Lifecycle -- **After PR merge**: Remove the issue file when its PR is merged -- **Batch cleanup**: Periodically clean up multiple closed issues during maintenance -- **Before releases**: Tidy documentation before major releases +Closed issue specs are **not deleted immediately**. They go through a two-stage lifecycle: -## Cleanup Approaches +1. **Stage 1 — Archive**: When an issue is closed, move its spec file from `docs/issues/` to + `docs/issues/closed/`. The file stays here as a reference buffer while adjacent issues are + still in progress. +2. **Stage 2 — Delete**: Once the spec is no longer referenced by active work (typically after + the next one or two related issues are also closed), delete it permanently. -### Option 1: Single Issue Cleanup (Recommended) +See [`docs/issues/closed/README.md`](../../../../docs/issues/closed/README.md) for the purpose +of the buffer folder. -1. Verify the issue is closed on GitHub -2. Remove the issue file from `docs/issues/` -3. Commit and push changes +## When to Archive (Stage 1) -### Option 2: Batch Cleanup +- **After PR merge**: Move the issue file when its PR is merged and the issue is closed. +- **Batch archive**: Periodically move multiple closed issue files during maintenance. +- **Before releases**: Tidy `docs/issues/` before major releases. -1. List all issue files in `docs/issues/` -2. Check status of each issue on GitHub -3. Remove all closed issue files -4. Commit and push with a descriptive message +## When to Delete (Stage 2) + +- The spec is no longer referenced by any open issue or active work. +- The related issue series has progressed far enough that the context is no longer needed. ## Step-by-Step Process @@ -36,7 +39,7 @@ metadata: **Single issue:** ```bash -gh issue view {issue-number} --json state --jq .state +gh issue view {issue-number} --repo torrust/torrust-tracker --json state --jq .state ``` Expected: `CLOSED` @@ -45,44 +48,46 @@ Expected: `CLOSED` ```bash for issue in 21 22 23 24; do - state=$(gh issue view "$issue" --json state --jq .state 2>/dev/null || echo "NOT_FOUND") - echo "$issue:$state" + state=$(gh issue view "$issue" --repo torrust/torrust-tracker --json state --jq .state 2>/dev/null || echo "NOT_FOUND") + echo "$issue: $state" done ``` -### Step 2: Remove Issue Documentation File +### Step 2: Move Issue File to `docs/issues/closed/` ```bash # Single issue -git rm docs/issues/42-add-peer-expiry-grace-period.md +git mv docs/issues/42-add-peer-expiry-grace-period.md docs/issues/closed/ # Batch -git rm docs/issues/21-some-old-issue.md \ - docs/issues/22-another-old-issue.md +git mv docs/issues/21-some-old-issue.md \ + docs/issues/22-another-old-issue.md \ + docs/issues/closed/ ``` ### Step 3: Commit and Push ```bash # Single issue -git commit -S -m "chore(issues): remove closed issue #42 documentation" +git commit -S -m "chore(issues): archive closed issue #42 spec to docs/issues/closed" # Batch -git commit -S -m "chore(issues): remove documentation for closed issues #21, #22, #23" +git commit -S -m "chore(issues): archive closed issue specs #21, #22, #23 to docs/issues/closed" git push {your-fork-remote} {branch} ``` -## Determining If an Issue File Should Stay - -Keep issue files when: +### Step 4 (Stage 2): Delete When No Longer Needed -- The issue is still open -- The PR is open (still being worked on) -- The specification is referenced from other active docs +```bash +git rm docs/issues/closed/42-add-peer-expiry-grace-period.md +git commit -S -m "chore(issues): remove closed issue #42 spec (no longer referenced)" +``` -Remove issue files when: +## Determining File Placement -- The issue is **closed** -- The implementing PR is **merged** -- The file is no longer referenced by active work +| Condition | Action | +| --------------------------------------- | ----------------------------- | +| Issue still open | Keep in `docs/issues/` | +| Issue closed, related work still active | Move to `docs/issues/closed/` | +| Issue closed, no longer referenced | Delete permanently | diff --git a/docs/issues/1697-ai-agent-configuration.md b/docs/issues/closed/1697-ai-agent-configuration.md similarity index 100% rename from docs/issues/1697-ai-agent-configuration.md rename to docs/issues/closed/1697-ai-agent-configuration.md diff --git a/docs/issues/1703-1525-01-persistence-test-coverage.md b/docs/issues/closed/1703-1525-01-persistence-test-coverage.md similarity index 100% rename from docs/issues/1703-1525-01-persistence-test-coverage.md rename to docs/issues/closed/1703-1525-01-persistence-test-coverage.md diff --git a/docs/issues/1706-1525-02-qbittorrent-e2e.md b/docs/issues/closed/1706-1525-02-qbittorrent-e2e.md similarity index 100% rename from docs/issues/1706-1525-02-qbittorrent-e2e.md rename to docs/issues/closed/1706-1525-02-qbittorrent-e2e.md diff --git a/docs/issues/1710-1525-03-persistence-benchmarking.md b/docs/issues/closed/1710-1525-03-persistence-benchmarking.md similarity index 100% rename from docs/issues/1710-1525-03-persistence-benchmarking.md rename to docs/issues/closed/1710-1525-03-persistence-benchmarking.md diff --git a/docs/issues/1713-1525-04-split-persistence-traits.md b/docs/issues/closed/1713-1525-04-split-persistence-traits.md similarity index 100% rename from docs/issues/1713-1525-04-split-persistence-traits.md rename to docs/issues/closed/1713-1525-04-split-persistence-traits.md diff --git a/docs/issues/1715-1525-04b-migrate-consumers-to-narrow-traits.md b/docs/issues/closed/1715-1525-04b-migrate-consumers-to-narrow-traits.md similarity index 100% rename from docs/issues/1715-1525-04b-migrate-consumers-to-narrow-traits.md rename to docs/issues/closed/1715-1525-04b-migrate-consumers-to-narrow-traits.md diff --git a/docs/issues/1717-1525-05-migrate-sqlite-and-mysql-to-sqlx.md b/docs/issues/closed/1717-1525-05-migrate-sqlite-and-mysql-to-sqlx.md similarity index 100% rename from docs/issues/1717-1525-05-migrate-sqlite-and-mysql-to-sqlx.md rename to docs/issues/closed/1717-1525-05-migrate-sqlite-and-mysql-to-sqlx.md diff --git a/docs/issues/1719-1525-06-introduce-schema-migrations.md b/docs/issues/closed/1719-1525-06-introduce-schema-migrations.md similarity index 100% rename from docs/issues/1719-1525-06-introduce-schema-migrations.md rename to docs/issues/closed/1719-1525-06-introduce-schema-migrations.md diff --git a/docs/issues/1721-1525-07-align-rust-and-db-types.md b/docs/issues/closed/1721-1525-07-align-rust-and-db-types.md similarity index 100% rename from docs/issues/1721-1525-07-align-rust-and-db-types.md rename to docs/issues/closed/1721-1525-07-align-rust-and-db-types.md diff --git a/docs/issues/523-internal-linting-tool.md b/docs/issues/closed/523-internal-linting-tool.md similarity index 100% rename from docs/issues/523-internal-linting-tool.md rename to docs/issues/closed/523-internal-linting-tool.md diff --git a/docs/issues/closed/README.md b/docs/issues/closed/README.md new file mode 100644 index 000000000..0d5e8b20a --- /dev/null +++ b/docs/issues/closed/README.md @@ -0,0 +1,23 @@ +# Recently Closed Issues + +This folder holds issue specification files for issues that have been closed but are kept +temporarily as a reference buffer for ongoing and upcoming work. + +## Purpose + +Closed spec files are moved here (rather than deleted immediately) because: + +- The reasoning and design decisions captured in a spec often remain relevant to the next + issue in a series. +- Reviewers and contributors benefit from being able to trace _why_ a decision was made + across multiple related issues. +- It provides a grace period before permanent removal, reducing the risk of losing context + that is still actively referenced. + +## Lifecycle + +1. **Issue closed / PR merged** → spec file moves from `docs/issues/` to `docs/issues/closed/`. +2. **Buffer period** → file lives here while adjacent issues are still in progress. +3. **Cleanup** → once the spec is no longer referenced by active work, it is deleted. + +Use the `cleanup-completed-issues` skill to manage this lifecycle. From 51998dd2b566ed532771ca1dda064266dd01ba61 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 12:16:44 +0100 Subject: [PATCH 19/22] fix(postgres): address Copilot PR review suggestions Three issues flagged by the Copilot reviewer on PR #1724: 1. Add PostgreSQL 14 to the db-compatibility CI matrix (.github/workflows/db-compatibility.yaml). The issue spec (1723-1525-08) lists 14 15 16 17 as the required version set; only 15-17 were present. 2. Add unit test for Driver::PostgreSQL in persistence_benchmark/reporting.rs. build_report handles PostgreSQL in the same match arm as MySQL but had no test coverage, leaving report-metadata regressions undetected. 3. Align create_legacy_pre_v4_schema to use BIGINT NOT NULL for keys.valid_until. Migration 1 creates the column as BIGINT NOT NULL; the helper used INTEGER NOT NULL, causing a type mismatch after migrator idempotency runs because there is no migration that widens that column. --- .github/workflows/db-compatibility.yaml | 2 +- .../bin/persistence_benchmark/reporting.rs | 23 +++++++++++++++++++ .../src/databases/driver/postgres/mod.rs | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/db-compatibility.yaml b/.github/workflows/db-compatibility.yaml index abbdf2c7d..def107c86 100644 --- a/.github/workflows/db-compatibility.yaml +++ b/.github/workflows/db-compatibility.yaml @@ -44,7 +44,7 @@ jobs: strategy: matrix: - postgres-version: ["15", "16", "17"] + postgres-version: ["14", "15", "16", "17"] steps: - id: checkout diff --git a/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs b/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs index e664d51f0..158a7662e 100644 --- a/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs +++ b/packages/tracker-core/src/bin/persistence_benchmark/reporting.rs @@ -81,4 +81,27 @@ mod tests { assert_eq!(report.meta.db_version, "8.4"); assert_eq!(report.meta.ops, 2); } + + #[test] + fn it_should_keep_postgresql_db_version_in_report_metadata() { + let db_version = DbVersion::from_str("17").expect("db version should parse"); + let timings_ms = ReportTimings { + benchmark: 5, + report_build: 1, + total: 6, + }; + let operation_stats = vec![OperationStats { + name: "load_keys".to_string(), + count: 1, + best: Duration::from_micros(1), + median: Duration::from_micros(2), + worst: Duration::from_micros(3), + }]; + + let report = build_report(&Driver::PostgreSQL, &db_version, 1, timings_ms, operation_stats); + + assert_eq!(report.meta.driver, "postgresql"); + assert_eq!(report.meta.db_version, "17"); + assert_eq!(report.meta.ops, 1); + } } diff --git a/packages/tracker-core/src/databases/driver/postgres/mod.rs b/packages/tracker-core/src/databases/driver/postgres/mod.rs index c51ff2ddc..8d1f441d0 100644 --- a/packages/tracker-core/src/databases/driver/postgres/mod.rs +++ b/packages/tracker-core/src/databases/driver/postgres/mod.rs @@ -265,7 +265,7 @@ mod tests { for stmt in [ "CREATE TABLE IF NOT EXISTS whitelist (id SERIAL PRIMARY KEY, info_hash VARCHAR(40) NOT NULL UNIQUE)", "CREATE TABLE IF NOT EXISTS torrents (id SERIAL PRIMARY KEY, info_hash VARCHAR(40) NOT NULL UNIQUE, completed INTEGER DEFAULT 0 NOT NULL)", - "CREATE TABLE IF NOT EXISTS keys (id SERIAL PRIMARY KEY, key VARCHAR(32) NOT NULL UNIQUE, valid_until INTEGER NOT NULL)", + "CREATE TABLE IF NOT EXISTS keys (id SERIAL PRIMARY KEY, key VARCHAR(32) NOT NULL UNIQUE, valid_until BIGINT NOT NULL)", "CREATE TABLE IF NOT EXISTS torrent_aggregate_metrics (id SERIAL PRIMARY KEY, metric_name VARCHAR(50) NOT NULL UNIQUE, value INTEGER DEFAULT 0 NOT NULL)", ] { ::sqlx::query(stmt).execute(pool).await.expect("schema DDL"); From 245e6a354a29426a1e610a12ed0e894f1b1aef10 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 12:25:52 +0100 Subject: [PATCH 20/22] ci(testing): collapse stable and nightly into matrix job Replace duplicated test-nightly and test-stable jobs with a single matrix-driven test job. Preserve behavior with matrix flags: - run formatting checks only for nightly - run qBittorrent E2E tests only for stable - keep toolchain-specific timeout values This removes duplicated setup/test steps while keeping the same CI coverage and execution policy. --- .github/workflows/testing.yaml | 85 +++++++++------------------------- 1 file changed, 23 insertions(+), 62 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index a30998b6a..ef775d6c2 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -8,10 +8,24 @@ env: CARGO_TERM_COLOR: always jobs: - test-nightly: - name: Test (nightly) + test: + name: Test (${{ matrix.toolchain }}) runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: ${{ matrix.timeout_minutes }} + + strategy: + matrix: + include: + - toolchain: nightly + components: rustfmt, clippy, llvm-tools-preview + timeout_minutes: 45 + run_format: true + run_qbittorrent_e2e: false + - toolchain: stable + components: clippy, llvm-tools-preview + timeout_minutes: 90 + run_format: false + run_qbittorrent_e2e: true steps: - id: checkout @@ -22,8 +36,8 @@ jobs: name: Setup Toolchain uses: dtolnay/rust-toolchain@stable with: - toolchain: nightly - components: rustfmt, clippy, llvm-tools-preview + toolchain: ${{ matrix.toolchain }} + components: ${{ matrix.components }} - id: node name: Setup Node.js @@ -51,6 +65,7 @@ jobs: - id: format name: Run Formatting-Checks + if: ${{ matrix.run_format }} run: cargo fmt --check - id: lint @@ -69,71 +84,17 @@ jobs: name: Run E2E Tests run: cargo run --bin e2e_tests_runner -- --config-toml-path "./share/default/config/tracker.e2e.container.sqlite3.toml" - test-stable: - name: Test (stable) - runs-on: ubuntu-latest - timeout-minutes: 90 - - steps: - - id: checkout - name: Checkout Repository - uses: actions/checkout@v6 - - - id: setup - name: Setup Toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - components: clippy, llvm-tools-preview - - - id: node - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version: "20" - - - id: cache - name: Enable Job Cache - uses: Swatinem/rust-cache@v2 - - - id: fetch - name: Download Dependencies - run: cargo fetch --verbose - - - id: linter - name: Install Internal Linter - run: cargo install --locked --git https://github.com/torrust/torrust-linting --bin linter - - - id: tools - name: Install Tools - uses: taiki-e/install-action@v2 - with: - tool: cargo-llvm-cov, cargo-nextest - - - id: lint - name: Run All Linters - run: linter all - - - id: test-docs - name: Run Documentation Tests - run: cargo test --doc --workspace - - - id: test - name: Run Unit Tests - run: cargo test --tests --benches --examples --workspace --all-targets --all-features - - - id: run-tracker-e2e-tests - name: Run E2E Tests - run: cargo run --bin e2e_tests_runner -- --config-toml-path "./share/default/config/tracker.e2e.container.sqlite3.toml" - - id: run-qbittorrent-e2e-test name: Run qBittorrent E2E Test (SQLite) + if: ${{ matrix.run_qbittorrent_e2e }} run: cargo run --bin qbittorrent_e2e_runner -- --compose-file ./compose.qbittorrent-e2e.sqlite3.yaml --timeout-seconds 600 - id: run-qbittorrent-e2e-test-mysql name: Run qBittorrent E2E Test (MySQL) + if: ${{ matrix.run_qbittorrent_e2e }} run: cargo run --bin qbittorrent_e2e_runner -- --db-driver mysql --timeout-seconds 600 - id: run-qbittorrent-e2e-test-postgresql name: Run qBittorrent E2E Test (PostgreSQL) + if: ${{ matrix.run_qbittorrent_e2e }} run: cargo run --bin qbittorrent_e2e_runner -- --db-driver postgresql --timeout-seconds 600 From 735fe1984dfc37d632a831cf22289e20b853157c Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 12:36:33 +0100 Subject: [PATCH 21/22] ci(testing): normalize sqlite qBittorrent e2e invocation Use --db-driver sqlite3 for the SQLite qBittorrent E2E step so it follows the same invocation pattern as MySQL and PostgreSQL. Behavior is unchanged because the runner maps sqlite3 to the same default compose file (compose.qbittorrent-e2e.sqlite3.yaml). --- .github/workflows/testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index ef775d6c2..2ad4735fc 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -87,7 +87,7 @@ jobs: - id: run-qbittorrent-e2e-test name: Run qBittorrent E2E Test (SQLite) if: ${{ matrix.run_qbittorrent_e2e }} - run: cargo run --bin qbittorrent_e2e_runner -- --compose-file ./compose.qbittorrent-e2e.sqlite3.yaml --timeout-seconds 600 + run: cargo run --bin qbittorrent_e2e_runner -- --db-driver sqlite3 --timeout-seconds 600 - id: run-qbittorrent-e2e-test-mysql name: Run qBittorrent E2E Test (MySQL) From 453dd486f9c5e0582f1e97297f478c65ecfcfd9d Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Fri, 1 May 2026 13:44:20 +0100 Subject: [PATCH 22/22] ci(testing): split unit checks and merge docker e2e flow Refactor testing workflow to optimize runtime while preserving docker cache reuse in e2e execution: - rename matrix job from test to unit - keep nightly/stable checks in unit (fmt/lint/docs/unit tests) - merge tracker e2e and qBittorrent e2e into a single docker-e2e job so tracker image build and docker layers are reused in the same runner - run qBittorrent scenarios sequentially for sqlite3/mysql/postgresql - set docker-e2e timeout to 90 minutes --- .github/workflows/testing.yaml | 35 ++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 2ad4735fc..6243da6c3 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -8,8 +8,8 @@ env: CARGO_TERM_COLOR: always jobs: - test: - name: Test (${{ matrix.toolchain }}) + unit: + name: Unit (${{ matrix.toolchain }}) runs-on: ubuntu-latest timeout-minutes: ${{ matrix.timeout_minutes }} @@ -20,12 +20,10 @@ jobs: components: rustfmt, clippy, llvm-tools-preview timeout_minutes: 45 run_format: true - run_qbittorrent_e2e: false - toolchain: stable components: clippy, llvm-tools-preview timeout_minutes: 90 run_format: false - run_qbittorrent_e2e: true steps: - id: checkout @@ -80,21 +78,42 @@ jobs: name: Run Unit Tests run: cargo test --tests --benches --examples --workspace --all-targets --all-features + docker-e2e: + name: Docker E2E + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v6 + + - id: setup + name: Setup Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - id: cache + name: Enable Job Cache + uses: Swatinem/rust-cache@v2 + + - id: fetch + name: Download Dependencies + run: cargo fetch --verbose + - id: run-tracker-e2e-tests name: Run E2E Tests run: cargo run --bin e2e_tests_runner -- --config-toml-path "./share/default/config/tracker.e2e.container.sqlite3.toml" - - id: run-qbittorrent-e2e-test + - id: run-qbittorrent-e2e-test-sqlite3 name: Run qBittorrent E2E Test (SQLite) - if: ${{ matrix.run_qbittorrent_e2e }} run: cargo run --bin qbittorrent_e2e_runner -- --db-driver sqlite3 --timeout-seconds 600 - id: run-qbittorrent-e2e-test-mysql name: Run qBittorrent E2E Test (MySQL) - if: ${{ matrix.run_qbittorrent_e2e }} run: cargo run --bin qbittorrent_e2e_runner -- --db-driver mysql --timeout-seconds 600 - id: run-qbittorrent-e2e-test-postgresql name: Run qBittorrent E2E Test (PostgreSQL) - if: ${{ matrix.run_qbittorrent_e2e }} run: cargo run --bin qbittorrent_e2e_runner -- --db-driver postgresql --timeout-seconds 600