From bf5bfe43fd4973fb560272337c2a44c6b2adc9eb Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Wed, 3 Jun 2026 12:37:12 +0100 Subject: [PATCH 1/2] chore(container): exclude irrelevant workspace members from nextest archive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1868. Add `--exclude workspace-coupling --exclude torrust-tracker-torrent-repository-benchmarking` to all four `cargo nextest archive` commands in the Containerfile (pre-link warm-up × 2 and final archive × 2, covering both debug and release builds). `cargo chef prepare` and `cargo chef cook` do not expose an `--exclude` flag (cargo-chef CLI limitation). The COPY/stub lines for the excluded packages are retained in the recipe stage because `cargo chef prepare` invokes `cargo metadata` internally, which requires every workspace member's manifest to be present on disk. This limitation and its rationale are documented in Containerfile comments. Issue spec (T1–T3 DONE, AC4 DONE): docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md --- Containerfile | 35 ++++++++++++++++--- .../ISSUE.md | 35 ++++++++++--------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/Containerfile b/Containerfile index f2cd2db00..57dba1ff1 100644 --- a/Containerfile +++ b/Containerfile @@ -52,6 +52,13 @@ WORKDIR /build/src COPY Cargo.toml Cargo.lock ./ COPY console/tracker-client/Cargo.toml console/tracker-client/ COPY contrib/bencode/Cargo.toml contrib/bencode/ +# workspace-coupling and torrust-tracker-torrent-repository-benchmarking are +# excluded from cargo nextest archive (see Cook and Build stages below), but +# their Cargo.toml manifests and stub source files must still be present here +# because `cargo chef prepare` uses `cargo metadata` internally to enumerate +# all workspace members, and `cargo metadata` aborts if any member's manifest +# or declared target file is missing. `cargo chef prepare` has no `--exclude` +# flag (only `--bin`), so these stubs cannot be omitted from the recipe stage. COPY contrib/dev-tools/analysis/workspace-coupling/Cargo.toml contrib/dev-tools/analysis/workspace-coupling/ COPY packages/axum-health-check-api-server/Cargo.toml packages/axum-health-check-api-server/ COPY packages/axum-http-server/Cargo.toml packages/axum-http-server/ @@ -185,34 +192,54 @@ RUN cargo chef prepare --recipe-path /build/recipe.json FROM chef AS dependencies_debug WORKDIR /build/src COPY --from=recipe /build/recipe.json /build/recipe.json +# Note: `cargo chef cook` does not support `--exclude` (the cargo-chef CLI only +# exposes `--workspace` and `--package`, not `--exclude`). The two irrelevant +# workspace members (workspace-coupling and torrust-tracker-torrent-repository- +# benchmarking) are therefore excluded only at the nextest archive stage below, +# where standard Cargo `--exclude` is supported. Their stubs are still compiled +# as part of the cook skeleton, but their unique transitive dependencies are not +# pulled into the final archive, which is where the build-time cost matters. RUN cargo chef cook --tests --workspace --all-features --recipe-path /build/recipe.json # Pre-link warm-up: Create and discard a nextest archive to warm up the linker # before final compilation. This improves incremental build cache efficiency # by pre-faulting the linker phases, avoiding redundant linking work in later stages. -RUN cargo nextest archive --tests --workspace --all-features --archive-file /build/temp.tar.zst && rm -f /build/temp.tar.zst +RUN cargo nextest archive --tests --workspace --all-features \ + --exclude workspace-coupling \ + --exclude torrust-tracker-torrent-repository-benchmarking \ + --archive-file /build/temp.tar.zst && rm -f /build/temp.tar.zst ## Cook (release) FROM chef AS dependencies WORKDIR /build/src COPY --from=recipe /build/recipe.json /build/recipe.json +# Note: `cargo chef cook` does not support `--exclude` — see Cook (debug) above. RUN cargo chef cook --tests --workspace --all-features --recipe-path /build/recipe.json --release # Pre-link warm-up: Create and discard a nextest archive to warm up the linker # before final compilation. This improves incremental build cache efficiency # by pre-faulting the linker phases, avoiding redundant linking work in later stages. -RUN cargo nextest archive --tests --workspace --all-features --archive-file /build/temp.tar.zst --release && rm -f /build/temp.tar.zst +RUN cargo nextest archive --tests --workspace --all-features \ + --exclude workspace-coupling \ + --exclude torrust-tracker-torrent-repository-benchmarking \ + --archive-file /build/temp.tar.zst --release && rm -f /build/temp.tar.zst ## Build Archive (debug) FROM dependencies_debug AS build_debug WORKDIR /build/src COPY . /build/src -RUN cargo nextest archive --tests --workspace --all-features --archive-file /build/torrust-tracker-debug.tar.zst +RUN cargo nextest archive --tests --workspace --all-features \ + --exclude workspace-coupling \ + --exclude torrust-tracker-torrent-repository-benchmarking \ + --archive-file /build/torrust-tracker-debug.tar.zst ## Build Archive (release) FROM dependencies AS build WORKDIR /build/src COPY . /build/src -RUN cargo nextest archive --tests --workspace --all-features --archive-file /build/torrust-tracker.tar.zst --release +RUN cargo nextest archive --tests --workspace --all-features \ + --exclude workspace-coupling \ + --exclude torrust-tracker-torrent-repository-benchmarking \ + --archive-file /build/torrust-tracker.tar.zst --release # Extract and Test (debug) diff --git a/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md b/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md index 3878d7f06..91b32abf8 100644 --- a/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md +++ b/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md @@ -92,12 +92,12 @@ significantly. Status values: `TODO`, `IN_PROGRESS`, `BLOCKED`, `DONE`. -| ID | Status | Task | Notes / Expected Output | -| --- | ------ | --------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| T1 | TODO | Verify `cargo nextest archive` supports `--exclude` | Confirm the flag works for cargo-nextest; check docs and local test. | -| T2 | TODO | Add `--exclude` flags to all Containerfile cargo commands | Six commands: `cargo chef cook` × 2 and `cargo nextest archive` × 4. Both excluded packages named explicitly. | -| T3 | TODO | Decide on `cargo chef prepare` exclusion | Determine if `--exclude` is supported/needed for `cargo chef prepare`. If yes, remove the COPY/stub entries for the two packages from the recipe stage. If no, document why the COPY/stub lines must stay. | -| T4 | TODO | Run full CI build and record timing evidence | CI log showing build time after exclusion. Compare against pre-fix baseline (19m03s build step, 38m total). | +| ID | Status | Task | Notes / Expected Output | +| --- | ------ | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| T1 | DONE | Verify `cargo nextest archive` supports `--exclude` | Confirmed: `--exclude` is standard Cargo package-selection syntax; `cargo nextest archive` passes it through to Cargo. Verified by inspecting cargo-nextest behaviour and Cargo docs. | +| T2 | DONE | Add `--exclude` flags to all Containerfile cargo commands | Applied to all 4 `cargo nextest archive` commands. `cargo chef cook` does **not** support `--exclude` (cargo-chef CLI limitation; see T3). Documented with comments in the Containerfile. | +| T3 | DONE | Decide on `cargo chef prepare` exclusion | Neither `cargo chef prepare` nor `cargo chef cook` exposes an `--exclude` flag. The COPY/stub lines for both excluded packages **must stay** so that `cargo metadata` (invoked by `prepare`) can resolve the workspace without missing manifest files. See Containerfile comment and AC4. | +| T4 | TODO | Run full CI build and record timing evidence | CI log showing build time after exclusion. Compare against pre-fix baseline (19m03s build step, 38m total). | ## Progress Tracking @@ -107,8 +107,8 @@ Status values: `TODO`, `IN_PROGRESS`, `BLOCKED`, `DONE`. - [x] Spec reviewed and approved by user/maintainer - [x] GitHub issue created and issue number added to this spec - [ ] (Optional, recommended for complex issues) Spec-only PR merged into `develop` before implementation -- [ ] Implementation completed -- [ ] Automatic verification completed (`linter all`, relevant tests, and any pre-push checks) +- [x] Implementation completed +- [x] Automatic verification completed (`linter all`, relevant tests, and any pre-push checks) - [ ] Manual verification scenarios executed and recorded (status + evidence) - [ ] Acceptance criteria reviewed after implementation and updated with evidence - [ ] Reviewer validated acceptance criteria and updated checkboxes @@ -119,15 +119,16 @@ Status values: `TODO`, `IN_PROGRESS`, `BLOCKED`, `DONE`. - 2026-06-03 00:00 UTC - GitHub Copilot - Drafted issue spec based on post-merge CI analysis of #1853 - draft file created - 2026-06-03 00:00 UTC - GitHub Copilot - Narrowed scope to `--exclude` fix only; layer split analysis moved to dependency-layer-cache-reuse draft - draft updated +- 2026-06-03 00:00 UTC - GitHub Copilot - Implemented: added `--exclude workspace-coupling --exclude torrust-tracker-torrent-repository-benchmarking` to all 4 `cargo nextest archive` commands in Containerfile; investigated and documented that `cargo chef cook` and `cargo chef prepare` do not support `--exclude` (cargo-chef CLI limitation); COPY/stub lines for excluded packages retained in recipe stage because `cargo chef prepare` invokes `cargo metadata` which requires all workspace manifests to be present ## Acceptance Criteria - [ ] AC1: `workspace-coupling` and `torrust-tracker-torrent-repository-benchmarking` do not appear in the container build compilation output. - [ ] AC2: The final `cargo nextest archive` step in the CI build completes in measurably less time than the 19m03s baseline recorded after #1853. - [ ] AC3: The tracker runtime image is produced correctly and all unit tests still pass inside the container build. -- [ ] AC4: The decision on `cargo chef prepare` exclusion is documented with rationale (either in the Containerfile comments or in this spec). -- [ ] `linter all` exits with code `0` -- [ ] Relevant tests pass +- [x] AC4: The decision on `cargo chef prepare` and `cargo chef cook` exclusion is documented: neither tool exposes `--exclude` in its CLI (cargo-chef limitation). `cargo chef prepare` uses `cargo metadata` internally, which requires every workspace member's manifest to exist on disk — the COPY/stub lines for the excluded packages are therefore retained in the recipe stage. `cargo chef cook` similarly has no `--exclude` flag; the exclusion is achieved entirely through the 4 `cargo nextest archive` commands where standard Cargo `--exclude` is supported. This is documented in Containerfile comments. +- [x] `linter all` exits with code `0` +- [x] Relevant tests pass - [ ] Manual verification scenarios are executed and documented (status + evidence) - [ ] Acceptance criteria are re-reviewed after implementation and reflect actual behavior - [ ] Documentation is updated when behavior/workflow changes @@ -154,12 +155,12 @@ Status values: `TODO`, `IN_PROGRESS`, `DONE`, `FAILED`, `BLOCKED`. ### Acceptance Verification -| AC ID | Status (`TODO`/`DONE`) | Evidence | -| ----- | ---------------------- | ------------------------------------ | -| AC1 | TODO | {CI log link} | -| AC2 | TODO | {timing comparison} | -| AC3 | TODO | {CI test stage log} | -| AC4 | TODO | {Containerfile comment or spec note} | +| AC ID | Status (`TODO`/`DONE`) | Evidence | +| ----- | ---------------------- | ----------------------------------------------------------------------------------- | +| AC1 | TODO | {CI log link} | +| AC2 | TODO | {timing comparison} | +| AC3 | TODO | {CI test stage log} | +| AC4 | DONE | See AC4 text above and Containerfile comments in the Cook (debug) and recipe stages | ## Risks and Trade-offs From baf6c8d1fbc74c9d204b0108dcd200f98826f4df Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Wed, 3 Jun 2026 13:12:21 +0100 Subject: [PATCH 2/2] chore(container): fix misleading cook-stage comment; clarify T2 task scope Address Copilot review comments on PR #1872: - Containerfile cook-stage comment: the previous wording implied that stub sources prevent the excluded packages' unique transitive dependencies from being compiled during the cook layer. In fact, dependency compilation is driven by Cargo.toml manifests (already in the recipe), not by stub sources. Reworded to make clear that the cook layer still compiles those packages, and that the build-time savings come from the archive/build stages where `--exclude` is applied to `cargo nextest archive`. - ISSUE.md T2 task label: renamed from "Add `--exclude` flags to all Containerfile cargo commands" to "Add `--exclude` flags to all `cargo nextest archive` commands in Containerfile" to match what was actually done (only the nextest archive commands were updated; chef cook/prepare cannot accept `--exclude`). --- Containerfile | 9 +++++---- .../ISSUE.md | 12 ++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Containerfile b/Containerfile index 57dba1ff1..976b95aea 100644 --- a/Containerfile +++ b/Containerfile @@ -195,10 +195,11 @@ COPY --from=recipe /build/recipe.json /build/recipe.json # Note: `cargo chef cook` does not support `--exclude` (the cargo-chef CLI only # exposes `--workspace` and `--package`, not `--exclude`). The two irrelevant # workspace members (workspace-coupling and torrust-tracker-torrent-repository- -# benchmarking) are therefore excluded only at the nextest archive stage below, -# where standard Cargo `--exclude` is supported. Their stubs are still compiled -# as part of the cook skeleton, but their unique transitive dependencies are not -# pulled into the final archive, which is where the build-time cost matters. +# benchmarking) are therefore still compiled as part of the cook skeleton +# (their Cargo.toml manifests are in the recipe, so cargo-chef cooks them). +# The build-time savings come from the archive/build stages: `cargo nextest +# archive` below is passed `--exclude` so those packages are not compiled from +# real source in the final archive. See Cook (release) and Build stages. RUN cargo chef cook --tests --workspace --all-features --recipe-path /build/recipe.json # Pre-link warm-up: Create and discard a nextest archive to warm up the linker # before final compilation. This improves incremental build cache efficiency diff --git a/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md b/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md index 91b32abf8..4c5837ade 100644 --- a/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md +++ b/docs/issues/open/1868-1840-workflow-performance-exclude-irrelevant-workspace-members/ISSUE.md @@ -92,12 +92,12 @@ significantly. Status values: `TODO`, `IN_PROGRESS`, `BLOCKED`, `DONE`. -| ID | Status | Task | Notes / Expected Output | -| --- | ------ | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| T1 | DONE | Verify `cargo nextest archive` supports `--exclude` | Confirmed: `--exclude` is standard Cargo package-selection syntax; `cargo nextest archive` passes it through to Cargo. Verified by inspecting cargo-nextest behaviour and Cargo docs. | -| T2 | DONE | Add `--exclude` flags to all Containerfile cargo commands | Applied to all 4 `cargo nextest archive` commands. `cargo chef cook` does **not** support `--exclude` (cargo-chef CLI limitation; see T3). Documented with comments in the Containerfile. | -| T3 | DONE | Decide on `cargo chef prepare` exclusion | Neither `cargo chef prepare` nor `cargo chef cook` exposes an `--exclude` flag. The COPY/stub lines for both excluded packages **must stay** so that `cargo metadata` (invoked by `prepare`) can resolve the workspace without missing manifest files. See Containerfile comment and AC4. | -| T4 | TODO | Run full CI build and record timing evidence | CI log showing build time after exclusion. Compare against pre-fix baseline (19m03s build step, 38m total). | +| ID | Status | Task | Notes / Expected Output | +| --- | ------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| T1 | DONE | Verify `cargo nextest archive` supports `--exclude` | Confirmed: `--exclude` is standard Cargo package-selection syntax; `cargo nextest archive` passes it through to Cargo. Verified by inspecting cargo-nextest behaviour and Cargo docs. | +| T2 | DONE | Add `--exclude` flags to all `cargo nextest archive` commands in Containerfile | Applied to all 4 `cargo nextest archive` commands. `cargo chef cook` does **not** support `--exclude` (cargo-chef CLI limitation; see T3). Documented with comments in the Containerfile. | +| T3 | DONE | Decide on `cargo chef prepare` exclusion | Neither `cargo chef prepare` nor `cargo chef cook` exposes an `--exclude` flag. The COPY/stub lines for both excluded packages **must stay** so that `cargo metadata` (invoked by `prepare`) can resolve the workspace without missing manifest files. See Containerfile comment and AC4. | +| T4 | TODO | Run full CI build and record timing evidence | CI log showing build time after exclusion. Compare against pre-fix baseline (19m03s build step, 38m total). | ## Progress Tracking