diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..4b7d20e --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" + +"$repo_root/scripts/pre-commit.sh" diff --git a/.github/agents/commiter.agent.md b/.github/agents/commiter.agent.md new file mode 100644 index 0000000..7747728 --- /dev/null +++ b/.github/agents/commiter.agent.md @@ -0,0 +1,51 @@ +--- +name: Committer +description: Proactive commit specialist for this repository. Use when asked to commit, prepare a commit, review staged changes before committing, write a commit message, run pre-commit checks, or create a signed Conventional Commit. +argument-hint: Describe what should be committed, any files to exclude, and whether the changes are already staged. +tools: [execute, read, search, edit, todo] +model: GPT-5 (copilot) +user-invocable: true +disable-model-invocation: false +--- + +You are the repository's commit specialist. Your job is to prepare safe, clean, +and reviewable commits for the current branch. + +Treat every commit request as a review-and-verify workflow, not as a blind +request to run `git commit`. + +## Repository Rules + +- Follow `AGENTS.md` for repository-wide behavior and `.github/skills/commit/skill.md` for commit-specific reference details. +- The pre-commit validation command is `./scripts/pre-commit.sh`. +- Create GPG-signed Conventional Commits. + +## Required Workflow + +1. Read the current branch, `git status`, and the staged or unstaged diff relevant to the request. +2. Summarize the intended commit scope before taking action. +3. Ensure the commit scope is coherent and does not accidentally mix unrelated changes. +4. Run `./scripts/pre-commit.sh` when feasible and fix issues that are directly related to the requested commit scope. +5. Propose a precise Conventional Commit message. +6. Create the commit with `git commit -S` only after the scope is clear and blockers are resolved. +7. After committing, run a quick verification check and report the resulting commit summary. + +## Constraints + +- Do not write code. +- Do not bypass failing checks without explicitly telling the user what failed. +- Do not rewrite or revert unrelated user changes. +- Do not create empty, vague, or non-conventional commit messages. +- Do not commit secrets, backup junk, or accidental files. +- **Do not mix skill/workflow documentation changes with implementation changes—always create separate commits.** + +## Output Format + +When handling a commit task, respond in this order: + +1. Commit scope summary +2. Blockers, anomalies, or risks +3. Checks run and results +4. Proposed commit message +5. Commit status +6. Post-commit verification diff --git a/.github/skills/check-udp-conntrack/skill.md b/.github/skills/check-udp-conntrack/skill.md new file mode 100644 index 0000000..ac262e3 --- /dev/null +++ b/.github/skills/check-udp-conntrack/skill.md @@ -0,0 +1,60 @@ +--- +name: check-udp-conntrack +description: Workflow for checking whether UDP packet loss or uptime degradation may be caused by conntrack saturation on the torrust-tracker-demo server. Use when diagnosing UDP timeouts, low newTrackon uptime, packet drops, conntrack pressure, UDP receive-buffer errors, or when validating whether conntrack tuning is still healthy. +metadata: + author: torrust + version: "1.0" +--- + + + +# Check UDP Conntrack + +## Overview + +Use this skill to investigate whether UDP instability is caused by kernel-side +conntrack saturation or related packet-path pressure. + +The canonical human-facing reference is: + +- `docs/udp-conntrack-runbook.md` + +Keep durable explanations and operational guidance in that document. This skill +should stay focused on workflow and safe execution. + +## When To Use + +Use this skill when the user asks to: + +- check whether conntrack is too small +- diagnose UDP timeouts or packet loss +- validate that current conntrack tuning is still active +- verify whether the server is dropping UDP packets +- assess whether current symptoms point to conntrack saturation or something else + +## Workflow + +1. Run the host checks from `docs/udp-conntrack-runbook.md`. +2. Summarize the results in terms of: + - conntrack occupancy + - presence or absence of `table full` events + - IPv4 and IPv6 UDP receive-buffer errors + - whether `NoPorts` counters are relevant or benign +3. Distinguish conntrack saturation from softirq/RX steering imbalance. +4. If the user asks to document the result, update the relevant issue evidence + or incident file and reference the runbook when appropriate. + +## Interpretation Rules + +- `nf_conntrack_count` near or equal to `nf_conntrack_max` means real pressure. +- Any fresh `nf_conntrack: table full, dropping packet` message is a confirmed problem. +- `UdpRcvbufErrors` or `Udp6RcvbufErrors` increasing during the incident means packet loss below the application layer. +- `NoPorts` counters alone do not prove tracker loss. +- High load average with one CPU dominated by `%soft` points to softirq concentration, not necessarily conntrack exhaustion. + +## Safety Constraints + +- Do not change sysctl values unless the user explicitly asks for a fix. +- If applying a fix, update both runtime state and persistent files when appropriate. +- Preserve issue-specific evidence in `docs/issues/evidence/ISSUE-/`. +- Do not present the skill as the primary source of truth; the runbook in `docs/` is the canonical explanation. diff --git a/.github/skills/commit/skill.md b/.github/skills/commit/skill.md index 04629cb..2477704 100644 --- a/.github/skills/commit/skill.md +++ b/.github/skills/commit/skill.md @@ -10,6 +10,17 @@ metadata: Always run the linters locally before committing to avoid CI failures. +## Commit Scope Separation + +**Critical rule**: Never mix skill/workflow documentation changes (`.github/skills/`, `.github/agents/`, AGENTS.md) with implementation changes in a single commit. + +When a change affects both documentation/skills and implementation: + +1. **First commit**: Update the skill/documentation files only (type: `docs`) +2. **Second commit**: Implement the feature or fix (type: `feat`, `fix`, etc.) + +This keeps changes logically separated and makes the commit history easier to review. + ## Prerequisites Install the linter binary once: @@ -23,17 +34,27 @@ Also requires: Node.js ≥ 20, `yamllint`, and `shellcheck` available on `$PATH` ## Run Linters Before Committing ```sh -./scripts/lint.sh +./scripts/pre-commit.sh ``` -This runs markdown, YAML, spell check, and shell script linters in sequence. -Stops on first failure. +This is the pre-commit entry point and currently delegates to `./scripts/lint.sh`. +It runs markdown, YAML, spell check, and shell script linters in sequence and +stops on first failure. + +The repository also provides a tracked Git hook at `.githooks/pre-commit`. +Enable it locally with: + +```sh +git config core.hooksPath .githooks +``` Fix all reported issues before committing. Add any new project-specific words to `project-words.txt` (one word per line). ## Commit Message Format +All commits must be GPG-signed. + Follow [Conventional Commits](https://www.conventionalcommits.org/): ```text @@ -43,3 +64,5 @@ Follow [Conventional Commits](https://www.conventionalcommits.org/): ``` Types: `feat`, `fix`, `docs`, `chore`, `refactor`. + +When updating documentation for skills or workflows, use `docs` type. Implementation changes use their appropriate type (`feat`, `fix`, etc.) in a separate commit. diff --git a/.github/skills/create-issue-branch/skill.md b/.github/skills/create-issue-branch/skill.md index 852c5a8..34fe46d 100644 --- a/.github/skills/create-issue-branch/skill.md +++ b/.github/skills/create-issue-branch/skill.md @@ -39,16 +39,15 @@ canonical title. Derive the normalized branch name from it. **Always show the branch name to the user and wait for explicit approval before creating it.** -### Step 3 — Create and switch to the branch +### Step 3 — Create and switch to the branch locally + +Create the branch locally only (do not push to remote at this stage): ```bash git checkout -b ``` -### Step 4 — Push the branch to origin (optional, do when first committing) - -```bash -git push -u origin -``` +### Step 4 — Push to remote when creating the pull request -The branch is typically pushed with the first commit rather than immediately after creation. +The branch is pushed to origin only when creating the pull request. Do not push the branch +to remote immediately after creation—wait until the PR workflow. diff --git a/.github/skills/open-github-issue/skill.md b/.github/skills/open-github-issue/skill.md index a88e8ab..c475b6f 100644 --- a/.github/skills/open-github-issue/skill.md +++ b/.github/skills/open-github-issue/skill.md @@ -3,7 +3,7 @@ name: open-github-issue description: Step-by-step process for creating and opening a GitHub issue in the torrust-tracker-demo repository. Use when asked to open, create, or file an issue. Covers writing the draft file, human review, opening on GitHub, renaming the file, and committing. Triggers on "open issue", "create issue", "new issue", "file issue", "draft issue". metadata: author: torrust - version: "1.0" + version: "1.2" --- # Opening a GitHub Issue @@ -22,16 +22,24 @@ Create the draft under `docs/issues/` using the naming conventions below. **File naming**: -- Ready to open soon: `docs/issues/ISSUE-NNN-short-description.md` (use `NNN` as placeholder) -- Long-running or complex: `docs/issues/drafts/short-description.md` (no issue number prefix) +- Always use a non-numbered draft path before publication: + `docs/issues/drafts/short-description.md` +- Do **not** create `docs/issues/ISSUE--...` before the GitHub issue exists. + +> **Important**: never guess or assume the issue number. The real number is only known +> after Step 4 (opening the issue on GitHub). Before Step 4: +> +> - Keep the draft in `docs/issues/drafts/`. +> - Use `**Issue**: _(to be filled after publication)_`. +> - Never write guessed links such as `[#14](...)` in draft content. **Draft file structure** (use Markdown, no fixed template required): ```markdown # -**Issue**: #NNN _(to be filled in after opening the GitHub issue)_ -**Related**: <links to related issues in this or other repos, if any> +**Issue**: _(to be filled after publication)_ +**Related**: <links to related issues in this or other repos — omit this line entirely if there are none> ## Overview @@ -46,12 +54,14 @@ Create the draft under `docs/issues/` using the naming conventions below. ### Step 2 — Run linters +Use the canonical lint script (see the `run-linters` skill for prerequisites and troubleshooting): + ```bash -npx markdownlint-cli2 "**/*.md" -npx cspell --no-progress +./scripts/lint.sh ``` -Fix any errors. Add new project-specific words to `project-words.txt` (one word per line). +Fix any errors. Add new project-specific words to `project-words.txt` (one word per line, keep the +file sorted). Re-run the linter after editing `project-words.txt` to confirm the errors are gone. ### Step 3 — Human review @@ -68,53 +78,51 @@ Use the GitHub API/tool to create the issue: - `docs:` documentation - `chore:` maintenance - **Body**: paste the draft content, omitting the frontmatter lines (`**Issue**: #NNN` and - `**Related**:` can be kept or adapted for GitHub; the `#NNN` marker should be updated to the - real issue number) -- Note the assigned issue number from the response + `**Related**:` can be kept or adapted for GitHub) +- Note the assigned issue number from the response. +- If opening is canceled or fails, do not invent a number and do not rename the draft file. + +> **Multiple issues in one session**: when one issue will reference another (e.g. a follow-up +> issue that links back to a root cause issue), open them in dependency order — open the +> referenced issue first, record its number, then open the referencing issue with the real link. +> Never guess or placeholder the number of an issue that has not been opened yet. ### Step 5 — Rename the draft file and update the issue link ```bash -# Rename: replace NNN with the real issue number -mv docs/issues/ISSUE-NNN-short-description.md docs/issues/ISSUE-<N>-short-description.md +# Move draft to canonical path only after GitHub assigns the number +mv docs/issues/drafts/short-description.md docs/issues/ISSUE-<N>-short-description.md ``` -Update the `**Issue**: #NNN` line inside the file to a full link: +Update the `**Issue**:` line inside the file to a full link, and update `**Related**:` if it +contains placeholder text: ```markdown **Issue**: [#<N>](https://github.com/torrust/torrust-tracker-demo/issues/<N>) ``` -If the draft was in `docs/issues/drafts/`, move it to `docs/issues/ISSUE-<N>-short-description.md`. - ### Step 6 — Lint again, then commit and push ```bash -npx markdownlint-cli2 "**/*.md" -npx cspell --no-progress +./scripts/lint.sh ``` ```bash +# Stage the issue file; also stage project-words.txt if new words were added git add docs/issues/ISSUE-<N>-short-description.md -git commit -m "docs: add draft issue for <short description> +git add project-words.txt # only if modified +git commit -m "docs: add issue file for <short description> Refs: #<N>" git push ``` -**Note**: if the `ISSUE-NNN-` file was previously committed (tracked by git), also stage its -deletion so git does not report it as an uncommitted change: - -```bash -git rm docs/issues/ISSUE-NNN-short-description.md -``` - ## Draft Location Decision -| Situation | Location | -| ----------------------------------------- | -------------------------------------------- | -| Simple issue, will be worked on soon | `docs/issues/ISSUE-NNN-short-description.md` | -| Complex / long-term / needs more research | `docs/issues/drafts/short-description.md` | +| Situation | Location | +| ------------------------------------------- | -------------------------------------------- | +| Any issue before publication | `docs/issues/drafts/short-description.md` | +| Published issue (number assigned by GitHub) | `docs/issues/ISSUE-<N>-short-description.md` | > Feature specifications and design documents are a separate process not yet defined for this > project. diff --git a/.github/skills/open-pull-request/skill.md b/.github/skills/open-pull-request/skill.md new file mode 100644 index 0000000..cd6aeeb --- /dev/null +++ b/.github/skills/open-pull-request/skill.md @@ -0,0 +1,88 @@ +--- +name: open-pull-request +description: Create and open a pull request in the torrust-tracker-demo repository. Use when asked to open a PR, create a pull request, submit a PR, or push changes and create a PR. Triggers on "open PR", "create PR", "submit PR", "push and open PR", "open pull request". +metadata: + author: torrust + version: "1.0" +--- + +# Opening a Pull Request + +This skill guides you through pushing your branch and creating a pull request on GitHub. + +## Prerequisites + +- Local commits already created and ready to push +- Branch created locally (see `create-issue-branch` skill for branch creation) +- GitHub CLI installed and authenticated + +## Workflow + +### Step 1 — Push the branch to remote + +Push the local branch to the remote repository: + +```bash +git push -u origin <branch-name> +``` + +The `-u` flag sets the upstream tracking branch. Git will output a link to create a PR. + +### Step 2 — Prepare the PR title and description + +**Title**: Follow Conventional Commits format with the issue type and scope + +**Examples**: + +- `feat(docker): update Docker images for security vulnerability fixes` +- `fix(tracker): resolve UDP socket binding issue` +- `docs: update deployment guide` + +**Description**: Should include: + +- Brief summary of the change +- Context or motivation (reference related issues or PRs) +- Changes made (list key files or components modified) +- Verification checklist (if applicable) +- Link to related issue using `Fixes #<issue-number>` or `Refs: #<issue-number>` + +### Step 3 — Create the pull request + +Use GitHub CLI to create the PR with title and description: + +```bash +gh pr create \ + --title "feat(scope): description" \ + --body "Description with Fixes #<issue-number>" \ + --base main \ + --head <branch-name> +``` + +The `--body` parameter supports markdown. Use `Fixes #<issue-number>` to auto-link and auto-close the issue when merged. + +### Step 4 — Verify the PR was created + +GitHub CLI will output the PR URL: + +```text +https://github.com/torrust/torrust-tracker-demo/pull/<number> +``` + +Open it to: + +- Review the commits +- Verify the issue is linked +- Check that CI/CD checks pass +- Monitor for review comments + +## Tips + +- **Link to issues**: Always include `Fixes #<issue-number>` in the PR body to auto-link +- **Review before pushing**: Run `git log --oneline -n <count>` to verify commits are correct +- **Check branch status**: Verify you're on the correct branch with `git status` before pushing +- **Wait for checks**: GitHub Actions will run linters and tests. Wait for them to pass before merging. + +## Related Skills + +- `create-issue-branch` — Creating a new branch for an issue +- `commit` — Committing changes to the repository diff --git a/.github/skills/researching-performance-problems/skill.md b/.github/skills/researching-performance-problems/skill.md new file mode 100644 index 0000000..3fc8a31 --- /dev/null +++ b/.github/skills/researching-performance-problems/skill.md @@ -0,0 +1,131 @@ +--- +name: researching-performance-problems +description: Workflow for investigating server and service performance bottlenecks in the torrust-tracker-demo repository. Use when debugging uptime degradation, high load, latency spikes, dropped packets, or suspected capacity limits. Triggers on "performance issue", "high load", "uptime drop", "bottleneck", "capacity", "scaling", "degraded service", "newTrackon uptime". +metadata: + author: torrust + version: "1.0" +--- + +<!-- cspell:ignore snmp nstat --> + +# Researching Performance Problems + +## Overview + +This skill provides a practical workflow to investigate performance and uptime +problems without jumping to conclusions too early. + +Use it to gather reproducible evidence, separate signal from noise, and decide +whether the root cause is load, network path issues, application behavior, or +infrastructure sizing. + +## When To Use + +Use this workflow when any of these symptoms appear: + +- Uptime drops (for example external probes below expected SLA) +- High load average or CPU saturation +- Increased request errors, timeouts, or dropped UDP traffic +- Suspected bottlenecks requiring tuning or scale-up decisions + +## Core Principles + +1. Capture evidence before changing infrastructure. +2. Keep raw artifacts in issue-scoped evidence folders. +3. Separate temporary conclusions from confirmed root cause. +4. Treat monitoring source differences explicitly (for example `tracker_stats` + vs `tracker_metrics`). +5. Avoid exposing secrets or client-identifying payloads in stored evidence. + +## Recommended Workflow + +### 1) Create Issue-Scoped Evidence Folder + +Use: + +- `docs/issues/evidence/ISSUE-<N>/` + +Add one file per capture with: + +1. Context +2. Exact commands +3. Raw output (sanitized if needed) +4. Short notes/findings + +### 2) Collect Baseline Host Snapshot + +Capture at minimum: + +- `date -u`, `uptime`, `free -h`, `df -h`, `ss -u -s` +- `docker compose ps` +- top CPU and memory process snapshots + +### 3) Collect Kernel and Network Pressure Signals + +Common checks: + +- UDP counters (`/proc/net/snmp`, `nstat`) +- Interface error/drops (`ip -s link`) +- Packet path checks during incidents (`tcpdump`) + +### 4) Collect Application and Service Signals + +- Service-level counters and errors from Prometheus +- Aggregated log category counts (avoid raw sensitive payload dumps) +- Container-level CPU/memory/network (`docker stats --no-stream`) + +### 5) Use Prometheus Deliberately + +- Validate scrape source mapping before analysis. +- Prefer counter-style series (`*_total`) for increases/rates. +- Use `query_range` for 24h-72h time-correlation, not only point-in-time queries. +- Explicitly document caveats when metric names suggest gauge-like behavior. + +### 6) Correlate with External Uptime + +Correlate internal metrics with external probe windows: + +- newTrackon status changes +- synthetic probes from multiple locations +- host/network pressure windows + +### 7) Decide on Remediation Path + +- Tune first if clear bottleneck is configuration-level. +- Scale up if pressure is persistent and tuning is insufficient. +- Validate impact after each change using same evidence method. + +## Common Things To Check + +1. CPU contention between reverse proxy and tracker process. +2. UDP receive buffer errors and queue/backlog pressure. +3. Request error ratio trend by protocol and port. +4. IPv4/IPv6 differences in failure behavior. +5. Restart windows vs uptime drops (restart alone rarely explains large drops). + +## Output Template (per investigation phase) + +Create a progress/conclusions note containing: + +1. What was done +2. What was learned +3. Temporary conclusions +4. Candidate actions (immediate, short-term tuning, scaling) +5. Open questions + +## Example From This Repository + +Use ISSUE-19 evidence as a reference implementation: + +- `docs/issues/evidence/ISSUE-19/2026-04-13-baseline-server-snapshot.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-host-kernel-tracker-sanitized-diagnostics.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-source-mapping.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-3h-summaries.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-progress-and-temporary-conclusions.md` + +## Safety Notes + +- Do not commit secrets, tokens, credentials, or private keys. +- Avoid storing raw logs with full client-identifying request payloads when not + strictly necessary. +- Prefer aggregated counts when raw logs are high-volume and privacy-sensitive. diff --git a/.github/skills/run-linters/skill.md b/.github/skills/run-linters/skill.md index 1e6524f..584c39c 100644 --- a/.github/skills/run-linters/skill.md +++ b/.github/skills/run-linters/skill.md @@ -1,6 +1,6 @@ --- name: run-linters -description: Run linters for the torrust-tracker-demo repository. Covers running all linters at once or a single linter. Triggers on "run linters", "lint", "check lint", "run lint", "linting", "spell check", "check markdown", "check yaml". +description: Run linters for the torrust-tracker-demo repository. Covers running the canonical lint script, installing the lint wrapper and prerequisites, and troubleshooting markdown, YAML, spelling, shellcheck, npm permission, and Node/cspell compatibility issues. Triggers on "run linters", "lint", "check lint", "run lint", "linting", "spell check", "check markdown", "check yaml", "lint script", "install linter", "lint troubleshooting". metadata: author: torrust version: "1.0" @@ -17,6 +17,8 @@ metadata: Runs markdown, YAML, spell check, and shell script linters in sequence. Stops on first failure. +This is the canonical repository lint command. + ## Run a Single Linter ```sh @@ -31,6 +33,21 @@ linter shellcheck # Shell scripts - **Spell check false positive**: add the word to `project-words.txt` (one word per line). - **Markdown error**: check the rule ID in the output against `.markdownlint.json`. - **YAML error**: check `.yamllint-ci.yml` for the relevant rule. +- **npm EACCES while installing tools**: use a user-local npm prefix: + + ```sh + export NPM_CONFIG_PREFIX="$HOME/.local" + export PATH="$HOME/.local/bin:$PATH" + ``` + +- **Unsupported NodeJS version from cspell**: either upgrade Node.js or install a + compatible cspell version explicitly, for example: + + ```sh + npm install -g cspell@8.17.5 + ``` + +- **`yamllint` or `shellcheck` missing**: install them before rerunning the script. ## Prerequisites @@ -41,3 +58,22 @@ cargo install torrust-linting --locked ``` Also requires `yamllint` and `shellcheck` on `$PATH`. + +Recommended install on Ubuntu: + +```sh +sudo apt-get update +sudo apt-get install -y yamllint shellcheck +``` + +If you want reliable non-root npm installs for the Node-based lint tools: + +```sh +export NPM_CONFIG_PREFIX="$HOME/.local" +export PATH="$HOME/.local/bin:$PATH" +npm install -g markdownlint-cli cspell@8.17.5 +``` + +## Reference + +For full repository-specific guidance, see [docs/linting.md](../../docs/linting.md). diff --git a/.github/skills/scale-up-server/skill.md b/.github/skills/scale-up-server/skill.md new file mode 100644 index 0000000..0a3c6fc --- /dev/null +++ b/.github/skills/scale-up-server/skill.md @@ -0,0 +1,206 @@ +--- +name: scale-up-server +description: Step-by-step workflow for resizing (scaling up) the Hetzner server in the torrust-tracker-demo stack. Use when asked to resize, scale up, or upgrade the server plan. Covers pre-resize preparation, graceful shutdown, provider panel action, post-resize recovery, and evidence capture. Triggers on "resize server", "scale up", "upgrade server plan", "Hetzner resize", "change server type". +metadata: + author: torrust + version: "1.0" +--- + +<!-- cspell:ignore nproc Rcvbuf snmp nstat urlencode --> + +# Scaling Up the Server + +## Overview + +This skill covers a **planned, live resize** of the Hetzner Cloud server: +shut down services gracefully, resize the instance in the provider panel, +restart services, and validate everything before re-opening to traffic. + +> **Important**: Resizing a Hetzner Cloud server **does not change IP addresses**. +> Neither the public IPv4/IPv6 addresses nor any attached Floating IPs are +> affected. DNS records and Floating IP assignments do not need updating. +> This is standard cloud-provider behavior for in-place resizes. + +## Responsibilities + +| Step | Who | +| ----------------------------------- | ---------------------- | +| Capture pre-resize baseline | AI assistant | +| Graceful service shutdown | AI assistant (via SSH) | +| Resize in Hetzner Cloud panel | **Human operator** | +| Post-resize recovery and validation | AI assistant (via SSH) | +| Document evidence and commit | AI assistant | + +--- + +## Workflow + +### Step 1 — Capture pre-resize baseline + +Before touching the server, record the current state so there is a before/after +reference. Save results to the issue-scoped evidence folder +(`docs/issues/evidence/ISSUE-<N>/00-pre-resize-baseline.md`). + +```bash +# Host snapshot +ssh demotracker 'date -u; nproc; free -h; uptime; df -h' + +# Docker services +ssh demotracker 'cd /opt/torrust && docker compose ps' + +# Prometheus request rates (5m window) +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" \ + --data-urlencode "query=sum(rate(http_tracker_core_requests_received_total{server_binding_protocol=\"http\",server_binding_port=\"7070\"}[5m]))"' + +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" \ + --data-urlencode "query=sum(rate(udp_tracker_server_requests_received_total{server_binding_protocol=\"udp\",server_binding_port=\"6969\"}[5m]))"' + +# UDP buffer error counters +ssh demotracker 'grep "^Udp:" /proc/net/snmp; nstat -az 2>/dev/null | grep -Ei "UdpRcvbufErrors|Udp6RcvbufErrors" || true' +``` + +Commit the baseline file before proceeding to shutdown. + +### Step 2 — Confirm readiness + +Before shutting down: + +- Baseline file is complete and committed. +- Branch is clean and pushed. +- Nightly backup window awareness (~03:00 UTC). Prefer resizing outside that window. +- Operator is available to complete the Hetzner panel action promptly. + +### Step 3 — Graceful service shutdown (AI assistant) + +Run from a local terminal. Capture the full output and record it in +`docs/issues/evidence/ISSUE-<N>/01-resize-execution.md`. + +```bash +ssh demotracker 'set -e + echo "=== shutdown-start-utc ===" + date -u +%Y-%m-%dT%H:%M:%SZ + cd /opt/torrust + echo "=== docker-compose-ps-before ===" + docker compose ps + echo "=== docker-compose-down ===" + docker compose down + echo "=== docker-compose-ps-after ===" + docker compose ps + echo "=== shutdown-end-utc ===" + date -u +%Y-%m-%dT%H:%M:%SZ' +``` + +Confirm all containers are stopped and networks are removed before handing over. + +### Step 4 — Resize in Hetzner Cloud panel (human operator) + +1. Log in to [Hetzner Cloud Console](https://console.hetzner.cloud/). +2. Navigate to the project and select the server (`torrust-tracker-demo` or similar). +3. Go to **Rescale** (or **Server type**) tab. +4. Select the target server type (e.g. CCX33) and confirm. +5. Wait for the resize to complete — typically under 2 minutes. +6. Power on the server if it does not start automatically. +7. Notify the AI assistant when the server is reachable again. + +> No IP address changes are required. Floating IPs, public IPs, and private +> network IPs all remain the same after a Hetzner in-place resize. + +### Step 5 — Post-resize recovery (AI assistant) + +Start all services and capture the new host profile: + +```bash +ssh demotracker 'set -e + echo "=== startup-utc ===" + date -u +%Y-%m-%dT%H:%M:%SZ + echo "=== host ===" + nproc; free -h; uptime + cd /opt/torrust + echo "=== docker-compose-up ===" + docker compose up -d + echo "=== docker-compose-ps ===" + docker compose ps' +``` + +### Step 6 — Post-resize validation (AI assistant) + +Run all checks and record outputs in the execution log. + +```bash +# Container health +ssh demotracker 'cd /opt/torrust && docker compose ps' + +# UDP buffer counters (should be zero after fresh boot) +ssh demotracker 'grep "^Udp:" /proc/net/snmp; nstat -az 2>/dev/null | grep -Ei "UdpRcvbufErrors|Udp6RcvbufErrors" || true' + +# Prometheus targets +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" \ + --data-urlencode "query=up{job=\"tracker_metrics\"}" + curl -sG "http://127.0.0.1:9090/api/v1/query" \ + --data-urlencode "query=up{job=\"tracker_stats\"}"' +``` + +External checks (from local machine): + +```bash +# HTTP tracker health +curl -fsS "https://http1.torrust-tracker-demo.com/health_check" + +# Grafana (302 to /login is expected) +curl -I "https://grafana.torrust-tracker-demo.com" + +# UDP port probe +nc -zvu udp1.torrust-tracker-demo.com 6969 2>&1 | head -5 +``` + +All services must reach `healthy` status, HTTP health must return `200`, +and Prometheus targets must show `up=1` before the resize is considered +complete. + +> The tracker API health endpoint (`/health_check` on `api.torrust-tracker-demo.com`) +> requires authentication and returns `500 unauthorized` without a token. +> This is expected and not a failure indicator. + +### Step 7 — Document and commit + +Fill in the execution log (`01-resize-execution.md`) with all checklist items, +the full timeline (start UTC / end UTC / total impact window), the command +outputs, and the validation results. + +Run linters before committing: + +```bash +./scripts/lint.sh +``` + +Commit with: + +```bash +git commit -S -m "docs(issue-<N>): document resize execution and post-resize validation" \ + -m "Refs: #<N>" +``` + +### Step 8 — Update infrastructure docs + +After the resize is confirmed stable: + +- Update the hardware table in `docs/infrastructure.md` to reflect the new + server type, vCPU count, RAM, storage, traffic allowance, and price. +- Add a row to `docs/infrastructure-resize-history.md` with the resize date, + old and new plan, throughput at resize time, normalized req/s per vCPU, + and a link to the related issue. + +--- + +## Post-Resize Observation Period + +After the resize, monitor for at least **7 days** before concluding success: + +- Fill one row per day in `docs/issues/evidence/ISSUE-<N>/02-post-resize-daily-checks.md` + using the same Prometheus queries from Step 1. +- Check external uptime from [newTrackon](https://newtrackon.com/) or similar. +- Watch UDP buffer error counters for any resurgence. + +Once the observation window is complete, fill the final comparison table in +`docs/issues/evidence/ISSUE-<N>/03-pre-post-comparison.md` and decide whether +the resize meets the acceptance criteria. diff --git a/AGENTS.md b/AGENTS.md index 91afd13..2e630b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,12 +14,58 @@ tracker-only; the index demo lives in a separate repository. For a more complete reference of the combined Index + Tracker setup, see the original [torrust/torrust-demo](https://github.com/torrust/torrust-demo) repo. +## Repository Structure + +### `server/` + +Contains **only files that are actually deployed on the live server**, mirroring +their exact paths (e.g. `server/etc/...`, `server/opt/...`). These are static +configuration files managed in version control and deployed to the server. + +Do **not** place generated artifacts, dashboards, or documentation here. +Files that live in application databases (e.g. Grafana dashboards stored in +Grafana's own database) must **not** go in `server/` even if they relate to +a server-side service. + +### `docs/` + +Documentation only: architecture decision records, issue notes, post-mortems, +and other reference material. Do **not** place backup exports here. + +### `backups/` + +Versioned backup exports of data managed by server-side applications +(i.e. data that lives in application databases, not in config files). +Examples: Grafana dashboards exported from the Grafana UI. +Organized by application: `backups/grafana/dashboards/`, etc. + +These backups are **not deployed to the server** — they exist solely for +recovery and sharing purposes. + ## Code Conventions +## Mutual Support And Proactivity + +These rules apply repository-wide to every assistant, including custom agents. + +When acting as an assistant in this repository: + +- Do not flatter the user or agree with weak ideas by default. +- Push back when a request, diff, or proposed commit looks wrong. +- Flag unclear but important points before they become problems. +- Ask a clarifying question instead of making a random choice when the decision matters. +- Call out likely misses such as naming inconsistencies, accidental generated files, + staged-versus-unstaged mismatches, missing docs updates, or suspicious commit scope. + +When raising a likely mistake or blocker, say so clearly and early instead of +burying it after routine status updates. + ### Commit Messages Use [Conventional Commits](https://www.conventionalcommits.org/): +- All commits must be GPG-signed. + - `feat:` — new features or configuration - `fix:` — bug fixes - `docs:` — documentation-only changes @@ -39,16 +85,45 @@ Format: ### Markdown - All Markdown files must pass the markdown linter. -- Run: `npx markdownlint-cli2 "**/*.md"` +- Preferred command: `./scripts/lint.sh` +- Direct command: `npx markdownlint-cli2 "**/*.md"` ### Spell Checking - All files must pass CSpell spell checking. -- Run: `npx cspell --no-progress` +- Preferred command: `./scripts/lint.sh` +- Direct command: `npx cspell --no-progress` - Add project-specific terms to `project-words.txt`. +### Linting Summary + +- The canonical lint entry point is `./scripts/lint.sh`. +- The pre-commit entry point is `./scripts/pre-commit.sh`. +- The repository includes a tracked Git hook at `.githooks/pre-commit` that runs the pre-commit script. +- Enable tracked hooks locally with `git config core.hooksPath .githooks`. +- Install the wrapper with `cargo install torrust-linting --locked`. +- Install `yamllint` and `shellcheck` on `$PATH` before running the script. +- If npm install steps fail with `EACCES`, use a user-local npm prefix. +- See [docs/linting.md](docs/linting.md) for installation and troubleshooting. + +### Commit Review Expectations + +Before creating a commit, review the diff like a skeptical reviewer, not a blind +operator. + +- Read `git status` and the relevant `git diff` first. +- Look for unusual states such as a file being staged and also deleted, mixed + unrelated changes, or files that do not fit repository naming patterns. +- Prefer stopping to clarify an anomaly over committing something ambiguous. +- If documentation, spelling word lists, or ignore rules should change with the + diff, call that out before committing. +- Use `./scripts/pre-commit.sh` as the commit-time validation command. + +After creating a commit, verify the result with a short `git status` check and +briefly summarize what was committed. + ## Pull Request Guidelines - Title must follow the Conventional Commits format: `<type>[scope]: <description>` -- Run the markdown linter and spell checker before opening a PR. +- Run `./scripts/pre-commit.sh` before opening a PR. - Reference related issues in the PR body using `Refs: #<issue>`. diff --git a/README.md b/README.md index 43ebaa0..3bc4097 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,31 @@ Live demo tracker endpoints: > The tracker is listed on [newtrackon](https://newtrackon.com/) for public uptime tracking. +![Newtrackon listing showing both trackers](docs/media/newtrackon-trackers.png) + +> **Note:** The low uptime shown for the UDP tracker (`udp1`) reflects the fact +> that it was added only 24 hours before this screenshot was taken, and there +> was an IPv6 routing misconfiguration during that period that prevented +> newtrackon from receiving responses. See the +> [post-mortem](docs/post-mortems/2026-03-09-udp-ipv6-docker.md) for details. +> Uptime figures should also be read in the context of server size and request +> load — a tracker running on a small server that is receiving more requests +> than it can handle will report low uptime even if the software itself is +> healthy. See [docs/infrastructure.md](docs/infrastructure.md) for the exact +> server specification used in this demo. + +## Grafana Dashboards + +Live public dashboards are available without a Grafana account: + +[![Grafana dashboards preview](docs/media/grafana-dashboards.webp)](https://grafana.torrust-tracker-demo.com) + +| Dashboard | Public link | Screenshot | +| ---------------- | --------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | +| Tracker Overview | [open](https://grafana.torrust-tracker-demo.com/public-dashboards/186b355b56cd482d9c441a0affdb8ecd) | [view](backups/grafana/dashboards/01-tracker-overview.png) | +| UDP Tracker 1 | [open](https://grafana.torrust-tracker-demo.com/public-dashboards/6d493d22396c4e3cbaeec5669ed2ae69) | [view](backups/grafana/dashboards/02-udp-tracker-1.png) | +| HTTP Tracker 1 | [open](https://grafana.torrust-tracker-demo.com/public-dashboards/ca57cd298a0240c5b7b7965b3c47ebcf) | [view](backups/grafana/dashboards/03-http-tracker-1.png) | + ## Background [Torrust](https://github.com/torrust) is an open-source organization building @@ -221,6 +246,28 @@ This runs markdown, YAML, spell check, and shell script linters in sequence. Add any new project-specific words to `project-words.txt` (one word per line) to suppress false positives from the spell checker. +## Disclaimer + +This demo tracker is provided **for documentation and educational purposes +only**. It is intended to demonstrate how to deploy and operate the +[Torrust Tracker](https://github.com/torrust/torrust-tracker), and not to +provide a persistent or general-purpose public tracking service. + +This software is provided solely for lawful purposes. Users must ensure +compliance with all applicable laws and regulations regarding copyright and +intellectual property. The Torrust organization and its contributors do not +condone or support the use of this tracker for any illegal activities, including +but not limited to the distribution of copyrighted, protected, or otherwise +illegal content. + +By using this tracker, you agree to use it responsibly and in compliance with +all applicable legal requirements. Misuse of this tracker for illegal purposes +may lead to legal consequences, for which the Torrust organization and its +contributors are not liable. + +**Tracker data (peer lists, announce history) may be reset at any time without +notice.** + ## Contributing Feedback, issues, and pull requests are welcome. If you spot a problem with the diff --git a/backups/grafana/dashboards/01-tracker-overview.json b/backups/grafana/dashboards/01-tracker-overview.json new file mode 100644 index 0000000..eead088 --- /dev/null +++ b/backups/grafana/dashboards/01-tracker-overview.json @@ -0,0 +1,511 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Global aggregate metrics across all protocols and IP families.\n\nData source: /api/v1/metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "tracker_core_persistent_torrents_downloads_total{job=\"tracker_metrics\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Completed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "swarm_coordination_registry_torrents_total{job=\"tracker_metrics\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Torrents", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "swarm_coordination_registry_peer_connections_total{job=\"tracker_metrics\", peer_role=\"seeder\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Seeders", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "swarm_coordination_registry_peer_connections_total{job=\"tracker_metrics\", peer_role=\"leecher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Leechers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "UDP announce requests per second across all UDP tracker instances (15m window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"announce\"}[15m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "UDP announces", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "UDP Announces (per sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "HTTP announce requests per second across all HTTP tracker instances (15m window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(http_tracker_core_requests_received_total{job=\"tracker_metrics\"}[15m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "HTTP announces", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "HTTP Announces (per sec)", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [ + "torrust", + "tracker" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Torrust Tracker - Overview", + "uid": "torrust-tracker-overview", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/backups/grafana/dashboards/01-tracker-overview.png b/backups/grafana/dashboards/01-tracker-overview.png new file mode 100644 index 0000000..3714b3f Binary files /dev/null and b/backups/grafana/dashboards/01-tracker-overview.png differ diff --git a/server/opt/torrust/storage/grafana/provisioning/dashboards/torrust/metrics.json b/backups/grafana/dashboards/02-udp-tracker-1.json similarity index 72% rename from server/opt/torrust/storage/grafana/provisioning/dashboards/torrust/metrics.json rename to backups/grafana/dashboards/02-udp-tracker-1.json index c95b981..aae611a 100644 --- a/server/opt/torrust/storage/grafana/provisioning/dashboards/torrust/metrics.json +++ b/backups/grafana/dashboards/02-udp-tracker-1.json @@ -15,11 +15,11 @@ } ] }, - "description": "Using metric endpoint:\n\nhttps://tracker.example.com/api/v1/metrics?token=MyAccessToken&format=prometheus", + "description": "Metrics for UDP Tracker 1 (port 6969). Filtered by server_binding_port=6969.\n\nData source: /api/v1/metrics", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": null, "links": [], "panels": [ { @@ -27,283 +27,7 @@ "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "tracker_core_persistent_torrents_downloads_total{job=\"tracker_metrics\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Completed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "swarm_coordination_registry_torrents_total{job=\"tracker_metrics\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Torrents", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "expr": "swarm_coordination_registry_peer_connections_total{job=\"tracker_metrics\", peer_role=\"seeder\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Seeders", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "swarm_coordination_registry_peer_connections_total{job=\"tracker_metrics\", peer_role=\"leecher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Leechers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Request per second in 15 minutes", + "description": "UDP connect requests per second (15m window)", "fieldConfig": { "defaults": { "color": { @@ -349,10 +73,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -360,12 +80,12 @@ "overrides": [] }, "gridPos": { - "h": 9, + "h": 7, "w": 6, "x": 0, - "y": 5 + "y": 0 }, - "id": 19, + "id": 1, "options": { "legend": { "calcs": [], @@ -381,18 +101,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"connect\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"connect\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Connections", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Connections (per sec)", + "title": "Connections (per sec)", "type": "timeseries" }, { @@ -400,7 +124,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Request per second in 15 minutes", + "description": "UDP announce requests per second (15m window)", "fieldConfig": { "defaults": { "color": { @@ -446,10 +170,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -457,12 +177,12 @@ "overrides": [] }, "gridPos": { - "h": 9, + "h": 7, "w": 6, "x": 6, - "y": 5 + "y": 0 }, - "id": 20, + "id": 2, "options": { "legend": { "calcs": [], @@ -478,18 +198,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"announce\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"announce\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Announces", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Announces (per sec)", + "title": "Announces (per sec)", "type": "timeseries" }, { @@ -497,7 +221,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Request per second in 15 minutes", + "description": "UDP scrape requests per second (15m window)", "fieldConfig": { "defaults": { "color": { @@ -543,10 +267,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -554,12 +274,12 @@ "overrides": [] }, "gridPos": { - "h": 9, + "h": 7, "w": 6, "x": 12, - "y": 5 + "y": 0 }, - "id": 21, + "id": 3, "options": { "legend": { "calcs": [], @@ -575,18 +295,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"scrape\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_requests_accepted_total{job=\"tracker_metrics\", request_kind=\"scrape\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Scrapes", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Scrapes (per sec)", + "title": "Scrapes (per sec)", "type": "timeseries" }, { @@ -594,7 +318,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Request per second in 15 minutes", + "description": "UDP errors per second (15m window)", "fieldConfig": { "defaults": { "color": { @@ -640,10 +364,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -651,12 +371,12 @@ "overrides": [] }, "gridPos": { - "h": 9, + "h": 7, "w": 6, "x": 18, - "y": 5 + "y": 0 }, - "id": 22, + "id": 4, "options": { "legend": { "calcs": [], @@ -672,18 +392,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_errors_total{job=\"tracker_metrics\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_errors_total{job=\"tracker_metrics\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Errors", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Errors (per sec)", + "title": "Errors (per sec)", "type": "timeseries" }, { @@ -691,7 +415,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "UDP4 Average Connect Processing Time", + "description": "Average connect request processing time in nanoseconds", "fieldConfig": { "defaults": { "color": { @@ -737,10 +461,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -749,12 +469,12 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 6, "x": 0, - "y": 14 + "y": 7 }, - "id": 17, + "id": 5, "options": { "legend": { "calcs": [], @@ -770,18 +490,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg(udp_tracker_server_performance_avg_processing_time_ns{job=\"tracker_metrics\", request_kind=\"connect\", server_binding_address_ip_family=\"inet\"})", + "expr": "avg(udp_tracker_server_performance_avg_processing_time_ns{job=\"tracker_metrics\", request_kind=\"connect\", server_binding_port=\"6969\"})", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Avg connect time", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Average Connect Time", + "title": "Avg Connect Time", "type": "timeseries" }, { @@ -789,7 +513,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "UDP4 Average Announce Processing Time", + "description": "Average announce request processing time in nanoseconds", "fieldConfig": { "defaults": { "color": { @@ -835,10 +559,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -847,12 +567,12 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 6, "x": 6, - "y": 14 + "y": 7 }, - "id": 16, + "id": 6, "options": { "legend": { "calcs": [], @@ -868,18 +588,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg(udp_tracker_server_performance_avg_processing_time_ns{job=\"tracker_metrics\", request_kind=\"announce\", server_binding_address_ip_family=\"inet\"})", + "expr": "avg(udp_tracker_server_performance_avg_processing_time_ns{job=\"tracker_metrics\", request_kind=\"announce\", server_binding_port=\"6969\"})", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Avg announce time", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Average Announce Time", + "title": "Avg Announce Time", "type": "timeseries" }, { @@ -887,7 +611,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "UDP4 Average Scrape Processing Time", + "description": "Average scrape request processing time in nanoseconds", "fieldConfig": { "defaults": { "color": { @@ -933,10 +657,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -945,12 +665,12 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 6, "x": 12, - "y": 14 + "y": 7 }, - "id": 18, + "id": 7, "options": { "legend": { "calcs": [], @@ -966,18 +686,22 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg(udp_tracker_server_performance_avg_processing_time_ns{job=\"tracker_metrics\", request_kind=\"scrape\", server_binding_address_ip_family=\"inet\"})", + "expr": "avg(udp_tracker_server_performance_avg_processing_time_ns{job=\"tracker_metrics\", request_kind=\"scrape\", server_binding_port=\"6969\"})", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Avg scrape time", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Average Scrape Time", + "title": "Avg Scrape Time", "type": "timeseries" }, { @@ -985,7 +709,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "UDP4 Banned Requests (per sec)", + "description": "Banned requests per second (15m window)", "fieldConfig": { "defaults": { "color": { @@ -1031,10 +755,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1042,12 +762,12 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 6, "x": 18, - "y": 14 + "y": 7 }, - "id": 14, + "id": 8, "options": { "legend": { "calcs": [], @@ -1069,18 +789,16 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_requests_banned_total{job=\"tracker_metrics\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_requests_banned_total{job=\"tracker_metrics\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", + "legendFormat": "Banned requests", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 Banned Requests (per sec)", + "title": "Banned Requests (per sec)", "type": "timeseries" }, { @@ -1088,7 +806,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "UDP 4 requests and responses", + "description": "Total requests received vs responses sent (15m window)", "fieldConfig": { "defaults": { "color": { @@ -1134,10 +852,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1145,10 +859,10 @@ "overrides": [] }, "gridPos": { - "h": 13, + "h": 18, "w": 18, "x": 0, - "y": 19 + "y": 14 }, "id": 9, "options": { @@ -1166,13 +880,16 @@ "pluginVersion": "11.4.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_requests_received_total{job=\"tracker_metrics\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_requests_received_total{job=\"tracker_metrics\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "legendFormat": "__auto", + "legendFormat": "Requests", "range": true, "refId": "A", "useBackend": false @@ -1184,18 +901,16 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_responses_sent_total{job=\"tracker_metrics\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_responses_sent_total{job=\"tracker_metrics\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", + "legendFormat": "Responses", "range": true, "refId": "B", "useBackend": false } ], - "title": "UDP4 Requests and Responses (per sec)", + "title": "Requests and Responses (per sec)", "type": "timeseries" }, { @@ -1203,7 +918,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "UDP Banned IPs", + "description": "Total number of currently banned IPs", "fieldConfig": { "defaults": { "color": { @@ -1249,10 +964,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1260,12 +971,12 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 9, "w": 6, "x": 18, - "y": 19 + "y": 14 }, - "id": 15, + "id": 10, "options": { "legend": { "calcs": [], @@ -1289,16 +1000,14 @@ "editorMode": "builder", "expr": "sum(udp_tracker_server_ips_banned_total{job=\"tracker_metrics\"})", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", + "legendFormat": "Banned IPs", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP Banned IPs", + "title": "Banned IPs", "type": "timeseries" }, { @@ -1306,7 +1015,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "", + "description": "Aborted requests per second (15m window)", "fieldConfig": { "defaults": { "color": { @@ -1352,10 +1061,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1363,12 +1068,12 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 9, "w": 6, "x": 18, - "y": 25 + "y": 23 }, - "id": 10, + "id": 11, "options": { "legend": { "calcs": [], @@ -1390,24 +1095,26 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(rate(udp_tracker_server_requests_aborted_total{job=\"tracker_metrics\", server_binding_address_ip_family=\"inet\"}[15m]))", + "expr": "sum(rate(udp_tracker_server_requests_aborted_total{job=\"tracker_metrics\", server_binding_port=\"6969\"}[15m]))", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", + "legendFormat": "Aborted requests", "range": true, "refId": "A", "useBackend": false } ], - "title": "UDP4 aborted requests (per second)", + "title": "Aborted Requests (per sec)", "type": "timeseries" } ], "preload": false, "schemaVersion": 40, - "tags": [], + "tags": [ + "torrust", + "tracker", + "udp" + ], "templating": { "list": [] }, @@ -1417,8 +1124,8 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Torrust Live Demo Tracker (metrics)", - "uid": "deogmiudufm68d", - "version": 50, + "title": "Torrust Tracker - UDP Tracker 1 (port 6969)", + "uid": "torrust-udp-tracker-1", + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/backups/grafana/dashboards/02-udp-tracker-1.png b/backups/grafana/dashboards/02-udp-tracker-1.png new file mode 100644 index 0000000..e2ab568 Binary files /dev/null and b/backups/grafana/dashboards/02-udp-tracker-1.png differ diff --git a/backups/grafana/dashboards/03-http-tracker-1.json b/backups/grafana/dashboards/03-http-tracker-1.json new file mode 100644 index 0000000..d6f3845 --- /dev/null +++ b/backups/grafana/dashboards/03-http-tracker-1.json @@ -0,0 +1,240 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics for HTTP Tracker 1 (port 7070). Filtered by server_binding_port=7070.\n\nData source: /api/v1/metrics\n\nNote: HTTP metrics are currently sparse. Only http_tracker_core_requests_received_total is confirmed exposed.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "HTTP announce requests per second (15m window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(http_tracker_core_requests_received_total{job=\"tracker_metrics\", request_kind=\"announce\", server_binding_port=\"7070\"}[15m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Announces", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Announces (per sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "HTTP scrape requests per second (15m window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(http_tracker_core_requests_received_total{job=\"tracker_metrics\", request_kind=\"scrape\", server_binding_port=\"7070\"}[15m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Scrapes", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Scrapes (per sec)", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [ + "torrust", + "tracker", + "http" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Torrust Tracker - HTTP Tracker 1 (port 7070)", + "uid": "torrust-http-tracker-1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/backups/grafana/dashboards/03-http-tracker-1.png b/backups/grafana/dashboards/03-http-tracker-1.png new file mode 100644 index 0000000..d9170a9 Binary files /dev/null and b/backups/grafana/dashboards/03-http-tracker-1.png differ diff --git a/docs/infrastructure-resize-history.md b/docs/infrastructure-resize-history.md new file mode 100644 index 0000000..73894a7 --- /dev/null +++ b/docs/infrastructure-resize-history.md @@ -0,0 +1,45 @@ +# Infrastructure Resize and Traffic History + +Tracks server sizing changes and observed traffic levels over time. + +Use this file to keep historical context for capacity decisions and uptime +investigations (especially for UDP uptime on newTrackon). + +## How to Use This Log + +1. Add one row before each resize (baseline). +2. Add one row after each resize (same observation method/time window). +3. Keep request-rate observations comparable (for example same dashboard window). +4. Link related issue/PR and note whether uptime improved. + +## Observation Method + +- HTTP request rate source: Grafana HTTP1 dashboard +- UDP request rate source: Grafana UDP1 dashboard +- Total request rate: `HTTP1 req/s + UDP1 req/s` +- Normalized load: `total req/s / vCPU` +- Suggested window: `from=now-3h` to `to=now` +- Uptime source: newTrackon public tracker status + +## Timeline + +| Date (UTC) | Change type | Server plan | vCPU | RAM | HTTP1 req/s | UDP1 req/s | Total req/s | Req/s per vCPU | UDP newTrackon uptime | Notes | Related | +| ---------- | --------------------- | ----------- | ---- | ----- | ----------- | ---------- | ----------- | -------------- | --------------------- | ----------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | +| 2026-04-13 | Baseline (pre-resize) | CCX23 | 4 | 16 GB | ~1350 | ~1507 | ~2857 | ~714 | 92.20% | Baseline from Prometheus 5m rate snapshot at 2026-04-13T15:27:46Z. Capacity pressure suspected. | [#19](https://github.com/torrust/torrust-tracker-demo/issues/19) | +| 2026-04-13 | Planned target resize | CCX33 | 8 | 32 GB | ~1350 | ~1507 | ~2857 | ~357 | 92.20% | Selected next plan: 30 TB traffic, €0.100/h - €62.49/mo. Assumes similar load after resize. | [#21](https://github.com/torrust/torrust-tracker-demo/issues/21) | + +## Decision Criteria (Suggested) + +- Target UDP uptime: >= 99.0% over a 7-day rolling window. +- Compare 3-7 days pre-resize vs 3-7 days post-resize. +- Consider resize successful if uptime improves materially and sustained error + pressure decreases. + +## Follow-up Checks After Each Resize + +1. Track UDP uptime daily for at least 7 days. +2. Re-check host load and UDP receive buffer errors. + For conntrack-specific diagnosis and remediation, use + [udp-conntrack-runbook.md](udp-conntrack-runbook.md). +3. Compare tracker error/aborted counters before vs after resize. +4. Record final conclusion in this file and in the related issue. diff --git a/docs/infrastructure.md b/docs/infrastructure.md index dac33eb..76423f8 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -4,6 +4,10 @@ Documents the hardware, network, and DNS configuration of the live demo server. For raw command outputs (`ip addr`, `df -h`, etc.) see [infrastructure-raw-outputs.md](infrastructure-raw-outputs.md). +For server resize and observed request-rate history see +[infrastructure-resize-history.md](infrastructure-resize-history.md). +For UDP packet-loss diagnosis and conntrack tuning guidance see +[udp-conntrack-runbook.md](udp-conntrack-runbook.md). ## Server @@ -18,7 +22,8 @@ For raw command outputs (`ip addr`, `df -h`, etc.) see | RAM | 16 GB | | Local disk | 160 GB NVMe SSD | | Volume | 50 GB, mounted at `/opt/torrust/storage` | -| Price | €23.99/month | +| Traffic | 20 TB | +| Price | €0.051/h - €31.49/month | | Datacenter | `nbg1-dc3` | | City | Nuremberg, Germany | | Network zone | `eu-central` | diff --git a/docs/issues/ISSUE-11-add-legal-disclaimer.md b/docs/issues/ISSUE-11-add-legal-disclaimer.md new file mode 100644 index 0000000..dd64e54 --- /dev/null +++ b/docs/issues/ISSUE-11-add-legal-disclaimer.md @@ -0,0 +1,80 @@ +# Add Legal Disclaimer to Protect the Torrust Org and Contributors + +**Issue**: [#11](https://github.com/torrust/torrust-tracker-demo/issues/11) +**Related**: +[torrust/torrust-index-gui](https://github.com/torrust/torrust-index-gui) — +contains a similar disclaimer that can serve as a starting point. + +## Overview + +The Torrust Tracker Demo runs a publicly accessible BitTorrent tracker. While +its purpose is purely educational — to demonstrate how to deploy and operate +the [Torrust Tracker](https://github.com/torrust/torrust-tracker) — the tracker +endpoints are reachable from the internet and can be used by anyone. + +BitTorrent trackers are a known target for legal actions in some jurisdictions, +because they can be used to coordinate the distribution of infringing content. +The Torrust Org and its contributors are not responsible for the content tracked +by the demo, and the demo is not intended to facilitate any illegal activity. +However, without an explicit disclaimer, this intent is not clear to users or +to third parties. + +This issue tracks the work needed to add a legal disclaimer that: + +- Makes clear that the demo is for documentation and educational purposes only. +- States that users are responsible for their own use of the tracker. +- States that the Torrust Org and contributors are not liable for misuse. +- Informs users that tracker data may be periodically reset. + +A follow-up issue should be opened to have the final disclaimer text reviewed +by a legal professional before the project reaches a larger audience. + +## Proposed Disclaimer Text + +The following text is proposed as an immediate first version, modelled on the +disclaimer already in use in the +[torrust/torrust-index-gui](https://github.com/torrust/torrust-index-gui) +repository: + +--- + +### Disclaimer + +This demo tracker is provided **for documentation and educational purposes +only**. It is intended to demonstrate how to deploy and operate the +[Torrust Tracker](https://github.com/torrust/torrust-tracker), and not to +provide a persistent or general-purpose public tracking service. + +This software is provided solely for lawful purposes. Users must ensure +compliance with all applicable laws and regulations regarding copyright and +intellectual property. The Torrust organization and its contributors do not +condone or support the use of this tracker for any illegal activities, including +but not limited to the distribution of copyrighted, protected, or otherwise +illegal content. + +By using this tracker, you agree to use it responsibly and in compliance with +all applicable legal requirements. Misuse of this tracker for illegal purposes +may lead to legal consequences, for which the Torrust organization and its +contributors are not liable. + +**Tracker data (peer lists, announce history) may be reset at any time without +notice.** + +--- + +## Implementation Plan + +- [ ] Add the disclaimer as a `## Disclaimer` section to `README.md`. +- [ ] Add the disclaimer text to `project-words.txt` for any technical terms + that fail the spell checker. +- [ ] Open a follow-up issue to have the final text reviewed by a legal + professional. + +## Acceptance Criteria + +- [ ] `README.md` contains a visible `## Disclaimer` section with the agreed + text. +- [ ] The repository passes the markdown linter (`npx markdownlint-cli2 +"**/*.md"`). +- [ ] The repository passes the spell checker (`npx cspell --no-progress`). +- [ ] A follow-up issue for legal review is referenced or opened. diff --git a/docs/issues/ISSUE-13-add-recurring-security-review-plan.md b/docs/issues/ISSUE-13-add-recurring-security-review-plan.md new file mode 100644 index 0000000..750c9ee --- /dev/null +++ b/docs/issues/ISSUE-13-add-recurring-security-review-plan.md @@ -0,0 +1,89 @@ +# Add a Recurring Security Review Plan for the Tracker Demo + +**Issue**: [#13](https://github.com/torrust/torrust-tracker-demo/issues/13) +**Related**: [docs/security/security-review-plan.md](../security/security-review-plan.md) + +## Overview + +The tracker demo is a public internet-facing deployment that exposes multiple +entry points, including HTTPS services, UDP tracker endpoints, SSH, and public +Grafana dashboards. The repository already documents infrastructure, +post-deployment steps, monitoring, and operational issues, but it does not yet +have a dedicated, repeatable security review process focused on realistic attack +paths to initial access. + +We need a maintained security review plan that can be reused periodically, +rather than a one-off note or an ad hoc checklist. The plan should make it easy +to reassess the same deployment over time, especially after infrastructure, +networking, authentication, or application changes. + +The review should answer a practical question: + +> How could an external attacker obtain meaningful access to the demo server or +> its deployed services? + +For this demo, meaningful access includes host access, privileged container +access, access to admin or sensitive application functionality, or access to +secrets and persistent state. + +## Why This Is Needed + +- The deployment is intentionally public and should be reviewed as an attacker + would see it. +- Security assumptions are currently spread across multiple files and are not + organized into a recurring review workflow. +- Changes to Docker networking, reverse proxy routing, Grafana exposure, + tracker API behavior, firewall rules, or image versions can change the attack + surface over time. +- A reusable plan lowers the cost of future reviews and makes the review scope + explicit for contributors. + +## Proposed Deliverable + +Add a dedicated security review planning document under `docs/security/` that +defines: + +- The review goal and scope. +- The review cadence. +- The current deployment surfaces that must always be reviewed. +- A phased review method covering configuration, source code, runtime + validation, and supply-chain review. +- A recurring checklist for future review cycles. +- An evidence request template listing the exact runtime and source information + needed for each review. +- The expected output of each review cycle. + +The document should be written as an operational reference, not as a single +incident note. + +## Implementation Plan + +- [ ] Create `docs/security/security-review-plan.md`. +- [ ] Document the review goal, scope, and recurring cadence. +- [ ] Document the main public entry points and trust boundaries currently + visible in the demo deployment. +- [ ] Define a phased review process covering external attack surface, source + review, host and container hardening, and supply-chain review. +- [ ] Add a recurring checklist for future review cycles. +- [ ] Add an evidence request template for the live environment and upstream + source repositories. +- [ ] Run the Markdown linter on the new documentation. +- [ ] Run the spell checker and add any legitimate project-specific words to + `project-words.txt`. + +## Acceptance Criteria + +- [ ] A security review plan exists at + `docs/security/security-review-plan.md`. +- [ ] The document is clearly written as a reusable and periodically reviewed + process document. +- [ ] The document includes review phases, recurring checklist items, and an + evidence request template. +- [ ] The document passes the Markdown linter. +- [ ] The document passes the spell checker. + +## Notes + +This issue covers the creation of the review plan itself. Actual security review +execution, findings, and follow-up fixes should be tracked in separate issue +documents or dated review notes that reference the plan. diff --git a/docs/issues/ISSUE-19-research-low-udp-uptime-on-newtrackon.md b/docs/issues/ISSUE-19-research-low-udp-uptime-on-newtrackon.md new file mode 100644 index 0000000..f34ef0a --- /dev/null +++ b/docs/issues/ISSUE-19-research-low-udp-uptime-on-newtrackon.md @@ -0,0 +1,114 @@ +# Research Low UDP Tracker Uptime on newTrackon + +**Issue**: [#19](https://github.com/torrust/torrust-tracker-demo/issues/19) +**Related**: +[torrust/torrust-demo#26](https://github.com/torrust/torrust-demo/issues/26), +[ISSUE-2](ISSUE-2-udp-tracker-down-on-newtrackon.md), +[ISSUE-5](ISSUE-5-external-monitoring.md) + +## Overview + +Current public uptime shown by [newTrackon](https://newtrackon.com/): + +- HTTP tracker: `https://http1.torrust-tracker-demo.com:443/announce` -> 99.90 % +- UDP tracker: `udp://udp1.torrust-tracker-demo.com:6969/announce` -> 92.20 % + +The HTTP endpoint is stable, but UDP uptime is significantly lower. This is not +the first time this pattern appears. A similar intermittent issue was previously +observed in the old demo setup: + +- [torrust/torrust-demo#26](https://github.com/torrust/torrust-demo/issues/26) + +At the time of writing, the old tracker demo UDP endpoint is healthy: + +- `udp://tracker.torrust-demo.com:6969/announce` -> 99.70 % + +This issue tracks a focused investigation to identify why the new demo has lower +UDP uptime and whether the root cause is resource saturation, network path +instability, runtime errors, or another operational bottleneck. + +## Working Hypotheses + +1. The server is occasionally resource-constrained (CPU, memory, network, disk + I/O), and UDP handling degrades during load spikes. +2. UDP packet handling on the host or Docker path is intermittently affected by + firewall/routing/NAT behavior. +3. Tracker runtime behavior under load (request bursts, database pressure, + socket pressure, queueing) causes transient failures that are visible in + external probes. +4. The issue is probe-specific or path-specific (newTrackon source paths, + IPv4/IPv6 asymmetry), while internal checks may still look healthy. + +## Investigation Plan + +### 1) Grafana and Prometheus evidence (current and historical load) + +- [ ] Review Grafana dashboards and Prometheus metrics around periods where + newTrackon UDP uptime dropped. +- [ ] Extract relevant time windows and values for: + CPU, memory, network throughput, packet drops/errors, disk I/O, + container restart counts, tracker request rates, and response latency. +- [ ] Correlate metric spikes with observed external UDP failures. + +### 2) Server usage and host telemetry + +- [ ] Capture host-level usage snapshots during normal load and peak load: + CPU, memory, swap, load average, open files, socket usage, network + interface stats, and bandwidth utilization. +- [ ] Check kernel/network counters for UDP drops and receive/send buffer + pressure. +- [ ] Record whether host limits (file descriptors, conntrack, buffers) are near + saturation. + +### 3) Network bottleneck analysis + +- [ ] Use network troubleshooting tools to inspect UDP flow behavior and packet + loss indicators. +- [ ] Validate firewall/routing/NAT behavior for UDP 6969 on both IPv4 and IPv6. +- [ ] Confirm there is no asymmetric routing or path-specific filtering. + +### 4) Docker Compose and tracker logs + +- [ ] Review Docker Compose service logs, focusing on the tracker service, for + warnings/errors during suspected failure windows. +- [ ] Check for container restarts, OOM kills, health-check flaps, and + dependency failures (database/cache/network). +- [ ] Look for tracker log patterns indicating overloaded sockets, timeout + spikes, or request-processing backlogs. + +### 5) Additional evidence collection + +- [ ] Compare behavior with the old demo endpoint to identify differences in + infrastructure, network setup, and load profile. +- [ ] Verify whether failures correlate with specific times of day or expected + traffic bursts. +- [ ] If feasible, run controlled announce load tests to reproduce degradation + and identify capacity boundaries. + +## Deliverables + +- [ ] A short incident-style report in `docs/post-mortems/` or + `docs/issues/` summarizing findings, timeline, and likely root cause(s). +- [ ] A prioritized action list with immediate mitigations and longer-term + fixes. +- [ ] If capacity is the bottleneck, a scaling plan (vertical sizing and/or + horizontal strategy) with expected impact. +- [ ] If observability is insufficient, a proposal for additional metrics, + dashboards, and alerts specific to UDP reliability. + +## Acceptance Criteria + +- [ ] Evidence from metrics, host telemetry, network diagnostics, and logs is + collected and documented. +- [ ] At least one probable root cause (or narrowed set of causes) is + identified with supporting data. +- [ ] Concrete remediation steps are defined and tracked in follow-up issues. +- [ ] A verification plan is defined to confirm improvement in newTrackon UDP + uptime after changes. + +## Notes + +Given recurring behavior and the difference between HTTP and UDP availability, +do not assume a single static configuration issue. Treat this as an +intermittent reliability investigation and prioritize correlation across data +sources over one-off spot checks. diff --git a/docs/issues/ISSUE-21-scale-up-server-for-udp-uptime.md b/docs/issues/ISSUE-21-scale-up-server-for-udp-uptime.md new file mode 100644 index 0000000..b31fa7b --- /dev/null +++ b/docs/issues/ISSUE-21-scale-up-server-for-udp-uptime.md @@ -0,0 +1,130 @@ +# Scale Up Demo Server Capacity to Improve UDP Tracker Uptime + +<!-- cspell:ignore Rcvbuf --> + +**Issue**: [#21](https://github.com/torrust/torrust-tracker-demo/issues/21) +**Related**: +[#19](https://github.com/torrust/torrust-tracker-demo/issues/19), +[infrastructure-resize-history.md](../../infrastructure-resize-history.md), +[2026-04-13-progress-and-temporary-conclusions.md](../evidence/ISSUE-19/2026-04-13-progress-and-temporary-conclusions.md) + +## Overview + +Observed traffic and evidence suggest the current server size (CCX23, 4 vCPU, +16 GB RAM) is likely under pressure for current request volume (about +1350 HTTP req/s + 1507 UDP req/s at the latest baseline snapshot). + +Current public uptime observed in newTrackon for UDP is below target: + +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> ~92% + +This issue tracks a controlled resize experiment to determine whether capacity +is the main bottleneck and to restore/maintain UDP uptime at or above 99%. + +## Current State (2026-04-27) — RESOLVED + +- Resize (CCX23 -> CCX33) complete and stable. +- Conntrack overflow root cause identified and fixed on 2026-04-20 + (`nf_conntrack_max` 262144 → 1048576, UDP timeouts reduced, module pre-load + added). +- 7-day post-fix observation window complete. +- newTrackon rolling UDP uptime reached **99.9%** — above the 99.0% target. + +Outcome: **Success**. See +[03-pre-post-comparison.md](evidence/ISSUE-21/03-pre-post-comparison.md) for +the final decision record. Permanent follow-up documentation now lives in +[udp-conntrack-runbook.md](../../udp-conntrack-runbook.md), with a reusable +workspace skill at `.github/skills/check-udp-conntrack/skill.md`. + +## Goal + +Increase UDP tracker uptime to at least 99.0% over a rolling 7-day window while +keeping service behavior stable. + +## Current Throughput Baseline (Pre-Resize) + +Observed request rates at baseline snapshot (`2026-04-13T15:27:46Z`): + +- Source: Prometheus instant query using 5-minute rate windows + +- HTTP1: ~1350 req/s +- UDP1: ~1507 req/s +- Combined: ~2857 req/s + +On the current CCX23 (4 vCPU), this is approximately: + +- ~714 req/s per vCPU (combined) + +This baseline must be preserved in the resize history so future sizing +decisions can be based on both absolute load and normalized load per vCPU. + +## Scope + +- Resize the demo server to a larger plan. +- Keep all other major changes constant during the observation window. +- Compare pre-resize and post-resize metrics and uptime. +- Record evidence in issue-scoped resize tracking files. + +## Selected Target Plan + +The next available option selected for this experiment is: + +| Property | Value | +| -------- | -------------------- | +| Plan | CCX33 | +| vCPUs | 8 (AMD) | +| RAM | 32 GB | +| SSD | 160 GB | +| Traffic | 30 TB | +| Price | €0.100/h - €62.49/mo | + +## Non-Goals + +- Redesigning the full architecture. +- Migrating services to multiple hosts. +- Making multiple tuning changes at the same time as resizing. + +## Implementation Plan + +1. Capture pre-resize baseline (request rates, UDP errors, host load, uptime). +2. Resize server from CCX23 to CCX33. +3. Verify service health after resize (docker services, tracker endpoints, + prometheus/grafana availability). +4. Observe and collect post-resize data for at least 7 days. +5. Compare before/after and decide whether resize is sufficient. +6. As part of this issue implementation, add a dedicated skill documenting the + server resize workflow and validation steps for future reuse. + +## Metrics and Evidence to Track + +- External uptime: + - newTrackon UDP uptime for `udp1` +- Traffic levels: + - HTTP1 request rate (Grafana) + - UDP1 request rate (Grafana) + - Combined request rate (HTTP1 + UDP1) + - Normalized load (combined req/s per vCPU) +- Reliability: + - `udp_tracker_server_errors_total` + - `udp_tracker_server_requests_aborted_total` + - `udp_tracker_server_responses_sent_total{result="error"}` +- Capacity pressure: + - Host load average + - Container CPU usage (tracker, caddy) + - UDP receive buffer errors (`UdpRcvbufErrors`, `Udp6RcvbufErrors`) + +## Acceptance Criteria + +- [x] Resize executed and documented in resize history. +- [x] No critical service regression immediately after resize. +- [x] At least 7 days of post-resize observations recorded. +- [x] UDP newTrackon uptime reaches and stays >= 99.0% during evaluation window. +- [x] Pre/post comparison documented with clear conclusion. +- [x] Resize workflow skill added and referenced. + +## Possible Outcomes + +- **Success**: Uptime >= 99% and error pressure decreases materially. +- **Partial**: Uptime improves but remains < 99%; continue with targeted tuning. +- **No improvement**: Capacity is not primary bottleneck; continue with + network/path/protocol-focused investigation. diff --git a/docs/issues/ISSUE-27-document-service-container-logs.md b/docs/issues/ISSUE-27-document-service-container-logs.md new file mode 100644 index 0000000..eaf42c6 --- /dev/null +++ b/docs/issues/ISSUE-27-document-service-container-logs.md @@ -0,0 +1,116 @@ +# Document Normal and Notable Service Container Logs + +**Issue**: [#27](https://github.com/torrust/torrust-tracker-demo/issues/27) + +## Overview + +While investigating unexpected log output from the Grafana container, we realised we had no +reference for what "normal" looks like for any of the services running in the demo. This made +it hard to judge whether a repeated log message was a real problem, a known quirk, or expected +behaviour. + +This issue proposes creating a `docs/logs/` section in the repository to document container log +output for each service: what is normal, what is noise, what signals a real problem, and why. +The goal is to avoid repeating the same investigation the next time we see an unfamiliar message. + +## Background: What Triggered This + +On 2026-04-20 we examined `docker logs grafana` and saw the following message repeating every +30 seconds: + +```text +INFO [04-20|...] No last resource version found, starting from scratch logger=dashboard-service orgID=1 +``` + +We did not know whether this was a bug, a misconfiguration, or normal. After research we +determined it is **expected behaviour in Grafana 12** introduced by the new Kubernetes-style +unified storage API (`dashboard-service`). Grafana watches this API for dashboard changes using +a resource version. When file-based provisioning is used (as in this demo), the storage backend +has no persistent resource version to return, so the watch resets every 30 seconds and logs this +message. Dashboards still load correctly from the file provider. + +No action is required for this message — but it took investigation time to confirm that. + +## Proposed Structure + +Create a `docs/logs/` directory organised by service: + +```text +docs/logs/ + README.md ← explains the purpose and how to read the sections + grafana.md ← Grafana container log reference + tracker.md ← Tracker container log reference (future) + prometheus.md ← Prometheus container log reference (future) + mysql.md ← MySQL container log reference (future) + caddy.md ← Caddy container log reference (future) +``` + +Each service file should cover: + +- **How to read the logs** — the command to run and what the log format looks like. +- **Normal steady-state messages** — messages that always appear and are safe to ignore. +- **Periodic housekeeping messages** — messages that appear on a schedule and what they mean. +- **Notable messages that need investigation** — patterns that indicate real problems. +- **Version notes** — behaviour that changed between versions. + +## Content for `docs/logs/grafana.md` (seed content from this investigation) + +### How to read Grafana logs + +```bash +ssh demotracker "docker logs grafana --tail 100 2>&1" +# or follow live: +ssh demotracker "docker logs grafana -f 2>&1" +``` + +Log format: `LEVEL [MM-DD|HH:MM:SS] <message> logger=<component> [key=value ...]` + +### Normal steady-state messages (Grafana 12.x) + +| Message | Frequency | Logger | Explanation | +| ------------------------------------------------------- | ------------ | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `No last resource version found, starting from scratch` | Every 30 s | `dashboard-service` | Grafana 12 unified storage API watch reset. Normal when using file-based provisioning. The watch finds no stored resource version and restarts from scratch. Dashboards load correctly. | +| `flag evaluation succeeded` | Every 10 min | `plugins.update.checker` | Plugin auto-update feature flag evaluated as `false` (disabled). Expected. | +| `Update check succeeded` | Every 10 min | `plugins.update.checker` | Grafana checked for plugin updates successfully. Expected. | +| `Completed cleanup jobs` | Every 10 min | `cleanup` | Routine database cleanup (sessions, temp files, etc.) completed. Expected. | + +### Periodic housekeeping messages (Grafana 12.x) + +| Message | Frequency | Logger | Explanation | +| --------------------------------- | ------------------- | ------------------ | ------------------------------------------------------------------ | +| `Building index using memory` | On demand / ~10 min | `bleve-backend` | Search index rebuilt for dashboards or folders. Normal. | +| `Finished building index` | On demand / ~10 min | `bleve-backend` | Confirms the index rebuild completed. | +| `Storing index in cache` | On demand / ~10 min | `bleve-backend` | Search index cached with a 10-minute TTL. | +| `index evicted from cache` | ~10 min | `bleve-backend` | Cache TTL expired; index will be rebuilt on next search. Normal. | +| `Usage stats are ready to report` | Periodic | `infra.usagestats` | Anonymous usage statistics prepared for reporting to Grafana Labs. | + +### Messages that warrant investigation + +| Pattern | Possible cause | +| --------------------------- | ----------------------------------------------------------- | +| `level=error` or `ERRO` | Any component error; read the full line | +| `level=warn` or `WARN` | Any component warning; may be ignorable, may not | +| `database is locked` | SQLite contention — check if multiple processes are writing | +| `failed to connect` | Data source (Prometheus) unreachable | +| `context deadline exceeded` | Query timeout; Prometheus may be overloaded or down | +| `provisioning failed` | Dashboard or datasource YAML has a syntax error | + +### Version notes + +- Grafana 12 introduced the unified storage API and the `dashboard-service` watch loop. + The `No last resource version found` message does **not** appear in Grafana 11 or earlier. + +## Implementation Plan + +- [ ] Create `docs/logs/README.md` explaining the section's purpose. +- [ ] Create `docs/logs/grafana.md` with the seed content from this investigation. +- [ ] Add placeholder stubs for `tracker.md`, `prometheus.md`, `mysql.md`, `caddy.md` + so future investigations have a clear home. + +## Acceptance Criteria + +- [ ] `docs/logs/` directory exists with a `README.md` and `grafana.md`. +- [ ] `grafana.md` documents at minimum: how to read logs, the `dashboard-service` watch + message, and the periodic housekeeping messages observed on 2026-04-20. +- [ ] Stub files exist for the remaining services. +- [ ] All files pass `./scripts/lint.sh`. diff --git a/docs/issues/ISSUE-28-harden-grafana-security-env-vars.md b/docs/issues/ISSUE-28-harden-grafana-security-env-vars.md new file mode 100644 index 0000000..d3ef2f8 --- /dev/null +++ b/docs/issues/ISSUE-28-harden-grafana-security-env-vars.md @@ -0,0 +1,126 @@ +# Harden Grafana Configuration with Missing Security Environment Variables + +**Issue**: [#28](https://github.com/torrust/torrust-tracker-demo/issues/28) +**Related**: [#27](https://github.com/torrust/torrust-tracker-demo/issues/27) + +## Overview + +During a Grafana log investigation on 2026-04-20 we audited the Grafana service configuration +in `server/opt/torrust/docker-compose.yml`. Comparing the current environment against the +[Grafana security hardening guide](https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-security-hardening/) +revealed several variables that should be explicitly set rather than left to their defaults. + +Grafana is served exclusively over HTTPS (Caddy handles TLS termination) with no OAuth or +SAML configured, so all the recommended hardening options apply without caveats. + +This issue tracks adding the missing variables and documenting the decisions made for each one. + +## Current State + +The Grafana service in `server/opt/torrust/docker-compose.yml` currently sets: + +```yaml +environment: + - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} + - GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL} +``` + +No security hardening variables are present. + +## Identified Gaps + +The following variables are absent and their defaults leave the instance less secure. + +### Cookie hardening + +| Variable | Default | Recommended | Reason | +| ----------------------------- | ------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GF_SECURITY_COOKIE_SECURE` | `false` | `true` | Sets the `Secure` attribute on the session cookie. Required when Grafana is behind an HTTPS reverse proxy; prevents the cookie from being sent over plaintext HTTP. | +| `GF_SECURITY_COOKIE_SAMESITE` | `lax` | `strict` | Sets the `SameSite=Strict` attribute to mitigate CSRF attacks. Safe to use when no OAuth or SAML login is configured (those require `lax`). | + +### Version disclosure + +| Variable | Default | Recommended | Reason | +| -------------------------------- | ------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `GF_AUTH_ANONYMOUS_HIDE_VERSION` | `false` | `true` | Hides the running Grafana version from unauthenticated users. Prevents trivial fingerprinting to find known-vulnerable versions. | + +### DNS rebinding protection + +| Variable | Default | Recommended | Reason | +| -------------------------- | ------- | ----------- | ------------------------------------------------------------------------------------------------------------- | +| `GF_SERVER_ENFORCE_DOMAIN` | `false` | `true` | Redirects requests whose `Host` header does not match the configured domain. Mitigates DNS rebinding attacks. | + +### Variables confirmed safe at their defaults + +| Variable | Default | Notes | +| ------------------------------ | ------- | --------------------------------------------- | +| `GF_USERS_ALLOW_SIGN_UP` | `false` | Self-registration disabled. No change needed. | +| `GF_AUTH_ANONYMOUS_ENABLED` | `false` | Anonymous access disabled. No change needed. | +| `GF_SECURITY_DISABLE_GRAVATAR` | `false` | Cosmetic only; no security impact. | + +### Variables to decide as part of this issue + +- `GF_SECURITY_CONTENT_SECURITY_POLICY` — Grafana's built-in CSP header. Caddy does not + set a CSP header by default, so this could be enabled. Needs testing to confirm Grafana + dashboards and plugins render correctly under the default CSP template. +- `GF_SECURITY_STRICT_TRANSPORT_SECURITY` — HSTS via Grafana. Caddy already sends HSTS + headers on the HTTPS listener, so enabling this in Grafana too is redundant. Recommend + leaving it disabled and documenting the decision. +- `GF_AUTH_LOGIN_COOKIE_NAME` with a `__Host-` prefix — cookie-prefix hardening (prevents + overwriting the session cookie in a MITM scenario even with HTTPS). More invasive change; + worth evaluating separately. +- Metrics endpoint auth (`GF_METRICS_BASIC_AUTH_USERNAME` / `GF_METRICS_BASIC_AUTH_PASSWORD`) + — the `/metrics` endpoint is accessible without auth by default. Needs confirming whether + this endpoint is reachable from outside the Docker network before deciding. + +## Proposed Change + +Add the confirmed variables to the Grafana service environment in +`server/opt/torrust/docker-compose.yml`: + +```yaml +environment: + - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} + - GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL} + - GF_SECURITY_COOKIE_SECURE=true + - GF_SECURITY_COOKIE_SAMESITE=strict + - GF_AUTH_ANONYMOUS_HIDE_VERSION=true + - GF_SERVER_ENFORCE_DOMAIN=true +``` + +After updating the file in the repository, apply the change to the live server: + +```bash +cd /opt/torrust +# pull updated docker-compose.yml from repo +docker compose up -d grafana +``` + +Grafana does not need a full image pull — `up -d` recreates the container with the new +environment and reconnects to the existing data volume. + +## Implementation Plan + +- [ ] Add `GF_SECURITY_COOKIE_SECURE=true` to the Grafana service env. +- [ ] Add `GF_SECURITY_COOKIE_SAMESITE=strict` to the Grafana service env. +- [ ] Add `GF_AUTH_ANONYMOUS_HIDE_VERSION=true` to the Grafana service env. +- [ ] Add `GF_SERVER_ENFORCE_DOMAIN=true` to the Grafana service env. +- [ ] Decide on `GF_SECURITY_CONTENT_SECURITY_POLICY` — test and document the decision. +- [ ] Decide on `GF_SECURITY_STRICT_TRANSPORT_SECURITY` — document the decision (likely: + leave disabled; Caddy already sends HSTS). +- [ ] Decide on cookie-prefix hardening and metrics endpoint auth — scope or defer. +- [ ] Apply the change to the live server (`docker compose up -d grafana`). +- [ ] Verify the session cookie attributes in browser DevTools + (Application → Cookies: `Secure`, `SameSite=Strict`). +- [ ] Confirm Grafana UI is fully functional after the change (dashboards, login, data sources). + +## Acceptance Criteria + +- [ ] All confirmed variables are present in `server/opt/torrust/docker-compose.yml`. +- [ ] The live server Grafana container is running with the new variables. +- [ ] The session cookie carries the `Secure` and `SameSite=Strict` attributes (verified + in browser DevTools). +- [ ] A documented decision exists for each "to decide" variable listed above. +- [ ] All changed files pass `./scripts/lint.sh`. diff --git a/docs/issues/ISSUE-29-research-high-cpu-load-after-udp-fix.md b/docs/issues/ISSUE-29-research-high-cpu-load-after-udp-fix.md new file mode 100644 index 0000000..b287f72 --- /dev/null +++ b/docs/issues/ISSUE-29-research-high-cpu-load-after-udp-fix.md @@ -0,0 +1,180 @@ +# Research high CPU load after UDP uptime recovery + +**Issue**: [#29](https://github.com/torrust/torrust-tracker-demo/issues/29) +**Related**: +[#21](https://github.com/torrust/torrust-tracker-demo/issues/21), +[#19](https://github.com/torrust/torrust-tracker-demo/issues/19) + +## Overview + +The UDP uptime problem tracked in [#21](https://github.com/torrust/torrust-tracker-demo/issues/21) +appears resolved: newTrackon is now healthy again after the conntrack fix and server resize. +However, the live server is still running at a high CPU load, which leaves limited headroom for +future traffic growth and may become the next reliability risk even before external uptime drops. + +This issue tracks a controlled investigation and remediation plan for the remaining CPU pressure. +The work should change only one variable at a time, with an observation period after each change, +so the impact of each action is measurable and attributable. + +## Current Baseline (2026-05-04) + +Live snapshot collected from the demo server on 2026-05-04: + +- Host load average: `8.50 / 8.27 / 8.20` +- Host CPU summary (`mpstat -P ALL 1 1`): + - all CPUs: `%usr=35.33`, `%sys=15.38`, `%soft=19.69`, `%idle=29.20` + - CPU2: `%soft=100.00`, `%idle=0.00` +- Top CPU consumers: + - `caddy`: about `279%` + - `torrust-tracker`: about `88%` + - `ksoftirqd/2`: visible in top CPU list +- Docker container CPU snapshot: + - `caddy`: `295.60%` + - `tracker`: `91.84%` + - `mysql`: `2.63%` + - `grafana`: `0.29%` + - `prometheus`: `0.00%` +- Current request rates from Prometheus: + - HTTP1: about `1982.32 req/s` + - UDP1: about `2124.28 req/s` + - Combined: about `4106.60 req/s` +- Conntrack state: + - `nf_conntrack_count`: `423120` + - `nf_conntrack_max`: `1048576` + - utilization: about `40.35%` +- RX steering state: + - `/sys/class/net/eth0/queues/rx-0/rps_cpus`: `00` + - `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt`: `0` + - `net.core.rps_sock_flow_entries`: `0` + +## What We Know So Far + +### Conntrack is no longer the immediate bottleneck + +The live conntrack table is well below the configured limit, so this does not look like a repeat +of the overflow problem fixed in [#21](https://github.com/torrust/torrust-tracker-demo/issues/21). + +### The packet path is still imbalanced on one CPU + +One CPU is saturated in softirq while the host still has idle capacity overall. That strongly +suggests packet processing is concentrated on one RX path instead of being distributed across +cores. + +The current live configuration confirms that RPS/RFS are disabled. + +### Caddy is also consuming several CPU cores + +This is not just a tracker-only problem. The HTTPS front-end is currently one of the largest CPU +consumers on the host. + +The current Compose file exposes UDP 443 for Caddy: + +```yaml +- "443:443/udp" +``` + +This is used for HTTP/3 over QUIC. It is not required for normal HTTPS over TCP. Standard HTTPS +would continue to work without it; removing it would only disable HTTP/3. + +The deployed server currently mirrors the repository on this point: the live +`/opt/torrust/docker-compose.yml` also exposes `443:443/udp`, and the host is listening on UDP 443. + +## Controlled Action Plan + +Important constraint: apply only one production change at a time and wait before taking the next +step. + +### Phase 1 — Preserve a reproducible baseline + +- [x] Record a baseline evidence snapshot under `docs/issues/evidence/ISSUE-29/` before making + any changes. See `00-baseline-live-snapshot.md` and `2026-05-04-htop-snapshot.png`. +- [x] Capture host load, per-CPU usage, top processes, docker stats, conntrack state, and RX + steering state. +- [x] Record at minimum the live HTTP1 and UDP1 request rates from Prometheus. +- [x] Record current newTrackon status for UDP1 and HTTP1. + +### Phase 2 — First isolated experiment: disable HTTP/3 + +- [x] Remove `"443:443/udp"` from the Caddy service in `server/opt/torrust/docker-compose.yml`. +- [x] Apply only that change on the live server and restart only Caddy. +- [x] Observe CPU, request rates, and external service health at T+1 h (≈ 2026-05-04 16:31 UTC). + **Result: no improvement. CPU2 still 100% softirq; Caddy ~321%; load ~8.5. HTTP/3 is not + the cause.** See `01-phase2-disable-http3-execution.md` T+1 h section. +- [x] Observe the following day (2026-05-05) to confirm no delayed effect. + **Result: no improvement. CPU2 still ~98% softirq; Caddy ~309%; load ~8.5. No delayed + effect.** See `01-phase2-disable-http3-execution.md` T+next-day section. +- [x] Decide whether Caddy CPU dropped materially enough to keep HTTP/3 disabled. + **Historical decision (2026-05-05): keep HTTP/3 disabled (hygiene). The change caused no + regression and removed an unused port mapping, but it did not reduce CPU load.** + **Update (2026-05-07): superseded by [#31](https://github.com/torrust/torrust-tracker-demo/issues/31), + which re-enables edge HTTP/3 as a product-capability choice with rollback triggers.** + +Clarification after [#31](https://github.com/torrust/torrust-tracker-demo/issues/31): + +- Disabling HTTP/3 did **not** reduce host CPU load or remove the softirq hotspot. +- Re-enabling `443:443/udp` was done to restore edge HTTP/3 capability for present and future + clients, not as a CPU-tuning change. +- Edge HTTP/3 on Caddy does **not** require native HTTP/3 support in backend services; Caddy can + terminate QUIC at the edge while continuing to proxy HTTP traffic to tracker and Grafana. + +Execution and immediate post-change checks are recorded in +`docs/issues/evidence/ISSUE-29/01-phase2-disable-http3-execution.md`. + +Rationale: this is a small, isolated change that affects only HTTP/3/QUIC support and does not +change normal HTTPS or the tracker's UDP listener on port 6969. + +### Phase 3 — Second isolated experiment: enable RPS/RFS + +- [x] If CPU pressure remains high after Phase 2, enable RPS and RFS as documented in + `docs/udp-conntrack-runbook.md`. +- [x] Persist the configuration in the tracked server config. +- [x] Re-check whether softirq is still concentrated on one CPU. + **Immediate result: improved.** CPU2 `%soft` dropped from `100%` to `48.51%` + and softirq work spread across all 8 CPUs. See + `docs/issues/evidence/ISSUE-29/02-phase3-enable-rps-rfs-execution.md`. +- [x] Observe for an agreed window before taking further action. + **T+1h (2026-05-05T09:13Z): distribution pattern stable. CPU2 %soft=49.48%, + no single-core saturation, both endpoints Working. T+next-day + (2026-05-06T09:24Z): distribution remains stable, both endpoints still + Working, but host load remains high (~11.83), indicating limited + headroom. See + `docs/issues/evidence/ISSUE-29/02-phase3-enable-rps-rfs-execution.md`.** + +Rationale: this directly targets the observed one-core softirq saturation while leaving the +application stack unchanged. + +### Phase 4 — Reassess architecture only after isolated tuning results exist + +- [ ] If both isolated experiments fail to provide enough headroom, evaluate moving the HTTPS + front-end or the UDP tracker onto separate hosts. +- [ ] Treat host separation as a later step, not the first response. + +## Questions To Answer In This Issue + +1. How much of the current CPU load is explained by higher traffic versus avoidable packet-path + inefficiency? +2. Does disabling HTTP/3 reduce Caddy CPU materially without unacceptable product impact? +3. Does enabling RPS/RFS spread softirq work enough to restore headroom? +4. After isolated tuning, is a single-host deployment still acceptable at current traffic? + +## Acceptance Criteria + +- [ ] A baseline evidence snapshot exists for the pre-change state, including HTTP1 and UDP1 + request rates. +- [ ] The first production change is isolated to a single variable and its effect is documented. +- [ ] No follow-up production change is applied before the previous change has been observed. +- [x] A documented decision exists on whether `443:443/udp` should remain enabled for HTTP/3. + **Historical decision: keep HTTP/3 disabled. Superseded by [#31](https://github.com/torrust/torrust-tracker-demo/issues/31) + (re-enable with controlled observation and rollback criteria).** +- [x] The issue records that disabling HTTP/3 did not materially reduce CPU load. + **Result: no meaningful CPU improvement was observed at T+1h or T+next-day checkpoints.** +- [x] The issue records why HTTP/3 was re-enabled. + **Reason: restore edge capability/future compatibility, with rollback triggers if resource + cost or availability regresses.** +- [x] The issue records that edge HTTP/3 is independent from backend native HTTP/3 support. + **Caddy terminates HTTP/3 at the edge and continues proxying HTTP to backend services.** +- [x] A documented decision exists on whether RPS/RFS should be deployed permanently. + **Decision: keep RPS/RFS enabled. It consistently removed the one-core + softirq hotspot at immediate, T+1h, and T+next-day checkpoints.** +- [ ] The final issue conclusion states whether the current single-host design still has enough + CPU headroom. diff --git a/docs/issues/ISSUE-30-scale-up-server-for-sustained-load.md b/docs/issues/ISSUE-30-scale-up-server-for-sustained-load.md new file mode 100644 index 0000000..15bda1d --- /dev/null +++ b/docs/issues/ISSUE-30-scale-up-server-for-sustained-load.md @@ -0,0 +1,223 @@ +# Scale Up Demo Server for Sustained Load Growth + +<!-- cspell:ignore CCX CCX13 CCX23 CCX33 CCX43 CCX53 CCX63 NVMe EX44 EX63 AX42 AX102 newtrackon --> + +**Issue**: [#30](https://github.com/torrust/torrust-tracker-demo/issues/30) +**Related**: +[#21](https://github.com/torrust/torrust-tracker-demo/issues/21), +[#29](https://github.com/torrust/torrust-tracker-demo/issues/29), +[infrastructure-resize-history.md](../../infrastructure-resize-history.md) + +## Overview + +After the conntrack fix and server resize in +[#21](https://github.com/torrust/torrust-tracker-demo/issues/21), UDP uptime +recovered to 99.9%. A follow-on softirq hotspot (CPU2 at 100% soft) was +diagnosed and fixed in [#29](https://github.com/torrust/torrust-tracker-demo/issues/29) +Phase 3 (RPS/RFS enabled on 2026-05-04). The hotspot was immediately resolved — +CPU soft-IRQ spread across all 8 CPUs — but the host load average remained in +the 9–10 range on an 8-vCPU machine. + +This means the server is running at or beyond its comfortable capacity even with +packet steering working correctly. If traffic continues to grow, or if the +softirq tuning is ever disrupted, uptime on newTrackon is likely to degrade +again. This issue tracks planning and execution of the next server scale-up. + +## Current State (2026-05-05) + +Post-RPS/RFS baseline collected at `2026-05-05T09:13:52Z` (T+1h after Phase 3): + +- Host load average: `9.24 / 9.24 / 9.43` (8-vCPU machine) +- CPU distribution (`mpstat -P ALL 1 1`): + - CPU2: `%soft=49.48` (was 100% before Phase 3) + - All CPUs: `%soft` in the 26–41% range + - Combined idle across all CPUs: low, indicating sustained saturation +- Docker container CPU (approximate): + - `caddy`: ~300% + - `tracker`: ~90% +- Request rates from Prometheus (5-minute rate): + - HTTP1: ~1982 req/s + - UDP1: ~2124 req/s + - Combined: ~4107 req/s (~513 req/s per vCPU) +- newTrackon status: both endpoints `Working` + +Load averages of 9–10 on an 8-vCPU host indicate the runqueue is consistently +overcommitted even after the softirq fix. There is very little headroom. + +## Goal + +Determine the right time and target plan to resize the server so that: + +- Host load average stays comfortably below the vCPU count (target: < 0.7 per + vCPU, i.e., < 5.6 on an 8-vCPU host or < 11.2 on a 16-vCPU host). +- Combined req/s per vCPU drops to a level that provides meaningful headroom. +- newTrackon uptime for both endpoints remains >= 99.0%. + +## Trigger Conditions + +**Do not resize until at least one of the following is true:** + +1. T+next-day observation in ISSUE-29 shows the RPS/RFS fix is not holding + (CPU2 `%soft` returns to 100% or overall soft-IRQ pressure re-concentrates). +2. newTrackon UDP uptime drops below 99.0% on the rolling 7-day window. +3. newTrackon HTTP uptime drops below 99.0%. +4. newTrackon response times for either endpoint increase materially (> 2× + current baseline) for more than 24 hours. +5. Load average exceeds 12 sustained over a 24-hour period (1.5× vCPU count). + +Track these signals in the **Observation Log** section below. + +## newTrackon Tracking + +Monitor both endpoints: + +- HTTP: `https://http1.torrust-tracker-demo.com:443/announce` +- UDP: `udp://udp1.torrust-tracker-demo.com:6969/announce` + +Check and record at each observation interval: + +- Status (Working / Down) +- Rolling uptime % +- Response time (ms) + +### Observation Log + +| Date (UTC) | HTTP1 status | HTTP1 uptime % | HTTP1 resp (ms) | UDP1 status | UDP1 uptime % | UDP1 resp (ms) | Load avg (1m) | Notes | +| ---------- | ------------ | -------------- | --------------- | ----------- | ------------- | -------------- | ------------- | --------------------------------- | +| 2026-05-05 | Working | — | — | Working | — | — | 9.24 | Baseline after RPS/RFS (ISSUE-29) | + +## Scope + +- Evaluate and select the next server plan (cloud or dedicated). +- Execute resize at the appropriate trigger point. +- Keep all other configuration constant during the observation window. +- Record pre-resize and post-resize metrics. +- Update `infrastructure-resize-history.md`. + +## Non-Goals + +- Migrating to a multi-host or distributed architecture. +- Making tuning changes simultaneously with the resize. +- Changing the tracker or proxy configuration. + +## Options Research + +All prices are list prices in EUR (excl. VAT) as of May 2026. + +### Hetzner Cloud — AMD Dedicated vCPU (CCX Series) + +These are cloud VMs with dedicated AMD vCPUs, easy to resize online via the +Hetzner console (no migration required, brief reboot only). + +| Plan | vCPU | RAM | NVMe SSD | Traffic | Price/month | +| ----- | ---- | ------ | -------- | ------- | ----------- | +| CCX13 | 2 | 8 GB | 80 GB | 20 TB | €16.49 | +| CCX23 | 4 | 16 GB | 160 GB | 20 TB | €31.99 | +| CCX33 | 8 | 32 GB | 240 GB | 30 TB | €62.99 | +| CCX43 | 16 | 64 GB | 360 GB | 40 TB | €125.49 | +| CCX53 | 32 | 128 GB | 600 GB | 40 TB | €250.49 | +| CCX63 | 48 | 192 GB | 960 GB | 60 TB | €374.99 | + +**Current plan: CCX33** (8 vCPU / 32 GB / €62.99/mo) + +**Next step up: CCX43** (16 vCPU / 64 GB / €125.49/mo — +€62.50/mo) + +CCX43 would reduce normalized load from ~513 req/s/vCPU to ~257 req/s/vCPU at +current traffic, and load average headroom would double. + +Advantages of cloud step-up: + +- No setup fee; no data migration. +- Revert is possible if the resize is not justified. +- Consistent experience with previous resize (CCX23 → CCX33). + +### Hetzner Dedicated Servers + +Dedicated physical servers provide more cores and threads per EUR, but require +a manual server migration (data copy, DNS/IP cutover) and a one-time setup fee. + +| Model | Cores | Threads | RAM | Storage | Bandwidth | Price/month | Setup fee | +| ------- | ----- | ------- | ------ | --------------- | --------- | ----------- | --------- | +| EX44 | 14 | 20 | 64 GB | 2 × 512 GB NVMe | 1000 Mbit | ~€44 | ~€109 | +| AX42-U | 8 | 16 | 64 GB | 2 × 512 GB NVMe | 1000 Mbit | ~€54 | ~€234 | +| EX63 | 20 | 20 | 64 GB | 2 × 1 TB NVMe | 1000 Mbit | ~€76 | ~€325 | +| AX102-U | 16 | 32 | 128 GB | varies | 1000 Mbit | ~€119 | ~€500 | + +**EX44 is the standout option** if we decide to go dedicated: + +- 14 physical cores / 20 threads vs 8 vCPUs today. +- 64 GB RAM (2× current). +- Monthly cost (~€44) is actually _cheaper_ than the current CCX33 (~€62.99). +- One-time setup fee of ~€109 is recovered in roughly 2 months of savings. +- Break-even vs CCX43 (~€125.49/mo): in month 1 total spend is ~€153 vs €125; + from month 2 onwards EX44 saves ~€82/mo over CCX43. + +Disadvantages of dedicated: + +- Manual migration required (bring-your-own IP, data copy, DNS update). +- No online resize; rollback is much harder. +- Bare-metal; OS and boot configuration is our responsibility. +- Physical hardware failure handling differs from cloud VMs. + +### Decision Matrix + +| Criterion | CCX43 (cloud step-up) | EX44 (dedicated) | +| ----------------------- | ----------------------- | ------------------------------------ | +| Monthly cost | €125.49 | ~€44 (saves ~€19/mo vs current) | +| Setup friction | Minimal (reboot only) | High (full migration) | +| Reversibility | Easy | Hard | +| CPU headroom at ~4k rps | 16 vCPU / ~257 rps/vCPU | 20 threads / ~205 rps/thread | +| RAM headroom | 64 GB | 64 GB | +| Long-term cost | More expensive | Cheaper after break-even (~2 months) | +| Risk | Low | Medium (migration complexity) | + +**Recommendation**: Start with CCX43 if the trigger is near-term and urgency +is high. Plan migration to EX44 if sustained long-term cost reduction is the +priority once the situation is stable. + +## Implementation Plan + +1. Monitor newTrackon and ISSUE-29 Phase 3 outcomes daily. +2. When a trigger condition is met, capture a pre-resize baseline snapshot + (request rates, load averages, uptime). +3. Select target plan based on urgency and risk tolerance (see decision matrix). +4. Execute resize following the procedure documented for ISSUE-21. +5. Verify all services recover post-resize. +6. Observe for at least 7 days and record post-resize metrics. +7. Update `infrastructure-resize-history.md` with pre- and post-resize rows. +8. Close this issue with a conclusion referencing the evidence. + +## Metrics and Evidence to Track + +- External uptime: + - newTrackon HTTP uptime for `http1` + - newTrackon UDP uptime for `udp1` + - newTrackon response times for both endpoints +- Traffic levels: + - HTTP1 request rate (Grafana) + - UDP1 request rate (Grafana) + - Combined request rate (HTTP1 + UDP1) + - Normalized load (combined req/s per vCPU) +- Capacity pressure: + - Host load average (1m, 5m, 15m) + - Container CPU usage (caddy, tracker) + - `%soft` per CPU from `mpstat` + +## Acceptance Criteria + +- [ ] Trigger condition documented and agreed upon before resize starts. +- [ ] Pre-resize baseline captured. +- [ ] Resize executed and documented in resize history. +- [ ] No critical service regression immediately after resize. +- [ ] At least 7 days of post-resize observations recorded. +- [ ] Host load average stays below vCPU count sustained (< 1.0 per vCPU). +- [ ] newTrackon uptime for both endpoints remains >= 99.0%. +- [ ] Pre/post comparison documented with clear conclusion. + +## Possible Outcomes + +- **Success**: Load average drops below vCPU count, uptime stable at >= 99.0%. +- **Partial**: Uptime holds but load remains high; additional tuning needed. +- **No improvement**: Traffic growth outpaces resize; larger plan required. +- **Premature**: ISSUE-29 Phase 3 outcome fully resolves headroom concerns; resize + deferred indefinitely. diff --git a/docs/issues/ISSUE-31-reenable-caddy-http3-and-document-rationale.md b/docs/issues/ISSUE-31-reenable-caddy-http3-and-document-rationale.md new file mode 100644 index 0000000..4444386 --- /dev/null +++ b/docs/issues/ISSUE-31-reenable-caddy-http3-and-document-rationale.md @@ -0,0 +1,121 @@ +# Re-enable Caddy HTTP/3 and Document ISSUE-29 Rationale + +**Issue**: [#31](https://github.com/torrust/torrust-tracker-demo/issues/31) +**Related**: +[#29](https://github.com/torrust/torrust-tracker-demo/issues/29), +[#30](https://github.com/torrust/torrust-tracker-demo/issues/30), +[ISSUE-29-research-high-cpu-load-after-udp-fix.md](ISSUE-29-research-high-cpu-load-after-udp-fix.md) + +## Overview + +ISSUE-29 removed Caddy UDP port mapping `443:443/udp` (HTTP/3 over QUIC) during a controlled +production experiment. The observations showed no measurable CPU improvement after disabling +HTTP/3, while service availability remained stable. + +This follow-up issue proposes re-enabling HTTP/3 at the Caddy edge and documenting why this +reversal is intentional. The goal is to restore HTTP/3 capability for present and future clients, +while keeping a controlled rollback path if resource cost or reliability regresses. + +This issue also clarifies the protocol boundary in the current architecture: + +- Edge protocol (client -> Caddy) can include HTTP/3. +- Backend protocol (Caddy -> tracker/grafana) remains reverse-proxy HTTP and does not require + tracker native HTTP/3 support. + +## Problem Statement + +The current ISSUE-29 wording can be read as "HTTP/3 disabled for hygiene" even though the +experiment outcome was only that disabling HTTP/3 did not fix CPU pressure. + +Keeping HTTP/3 disabled by default may also block automatic support for clients that prefer +or require HTTP/3 in the future. Since this demo already runs a Caddy edge proxy, re-enabling +UDP 443 is a low-complexity way to restore HTTP/3 capability without changing backend services. + +The change should therefore be treated as a product-capability decision with operational +guardrails, not as a CPU-remediation tactic. + +## Goals + +1. Re-enable Caddy UDP 443 publish mapping for HTTP/3 at the edge. +2. Keep backend application topology unchanged. +3. Record explicitly that ISSUE-29 did not show CPU benefit from disabling HTTP/3. +4. Document why re-enable is being done now (future compatibility/capability) and how rollback + will be handled if needed. + +## Proposed Change + +1. Re-add `"443:443/udp"` in Caddy service ports in + `server/opt/torrust/docker-compose.yml`. +2. Apply the same change on live `/opt/torrust/docker-compose.yml` and recreate only Caddy. +3. Observe immediate, T+1h, and T+next-day checkpoints with the same metrics used in ISSUE-29. + +## Rollback Triggers + +If any trigger is met after re-enable, revert by removing `"443:443/udp"` again and record +the rollback in evidence: + +1. Caddy CPU increases by more than 20% sustained for 24h vs pre-change baseline. +2. Host load average increases by more than 15% sustained for 24h vs pre-change baseline. +3. New external availability regression appears on tracked HTTP1 or UDP1 endpoints. + +## Deliverables + +- Compose change that re-enables Caddy UDP 443 publish mapping. +- Evidence notes for post-change observations (immediate, T+1h, T+next-day). +- Updated ISSUE-29 wording that clearly separates: + - measured performance result, + - capability/product decision, + - rollback criteria and operational safeguards. + +## Execution Status + +- Repository config change completed: Caddy UDP 443 mapping has been re-added in + `server/opt/torrust/docker-compose.yml`. +- Live-server Caddy recreate completed and UDP 443 listener validation completed. +- Immediate post-change evidence captured in + `docs/issues/evidence/ISSUE-31/00-immediate-post-change-snapshot.md`. +- T+1h checkpoint captured in + `docs/issues/evidence/ISSUE-31/01-t1h-snapshot.md`. +- T+next-day checkpoint captured in + `docs/issues/evidence/ISSUE-31/02-next-day-snapshot.md`. +- Strict sustained-24h CPU/load trigger evaluation is currently inconclusive due + to host restart between checkpoints (new uptime window). +- ISSUE-29 has been updated to record the re-enable rationale and clarify that + edge HTTP/3 does not require backend native HTTP/3 support. + +## Implementation Plan + +- [x] Re-add `"443:443/udp"` for Caddy in `server/opt/torrust/docker-compose.yml`. +- [x] Apply only that change on the live server and recreate only Caddy. +- [x] Validate Caddy health and confirm host UDP 443 listener exists after deploy. +- [x] Capture immediate post-change metrics: `mpstat`, `docker stats`, Prometheus HTTP1/UDP1 + rates, and `newtrackon.com/raw` sample. +- [x] Capture T+next-day checkpoint with the same metrics. +- [x] Evaluate rollback triggers; if triggered, revert and record evidence. +- [x] Update ISSUE-29 text to explain why the earlier disablement is being reversed now. +- [x] Ensure ISSUE-29 states backend services do not need native HTTP/3 for edge HTTP/3 support. +- [x] Run `./scripts/lint.sh` and fix any markdown/cspell issues. + +## Acceptance Criteria + +- [x] Caddy HTTP/3 edge capability is re-enabled via `443:443/udp` mapping. +- [x] Immediate, T+1h, and T+next-day evidence snapshots are recorded. +- [x] No rollback trigger is met during the observation window, or rollback is executed and + documented if a trigger is met. +- [x] ISSUE-29 explicitly states that disabling HTTP/3 did not reduce CPU in prior observations. +- [x] ISSUE-29 explicitly states why HTTP/3 was re-enabled and under which conditions it may be + disabled again. +- [x] Documentation clearly states edge HTTP/3 is independent from backend native HTTP/3 support. +- [x] All changed files pass `./scripts/lint.sh`. + +## Conclusion + +ISSUE-31 can be closed. + +- Edge HTTP/3 support has been restored by re-enabling `443:443/udp` on Caddy. +- Immediate, T+1h, and T+next-day evidence shows stable service health and no + observed availability regression. +- A strict sustained-24h CPU/load comparison was interrupted by a host restart, + so that specific continuity requirement is inconclusive rather than failed. +- No rollback trigger was met in the observed checkpoints, so rollback is not + indicated. diff --git a/docs/issues/ISSUE-9-grafana-public-dashboard-url-uses-localhost.md b/docs/issues/ISSUE-9-grafana-public-dashboard-url-uses-localhost.md new file mode 100644 index 0000000..4716b33 --- /dev/null +++ b/docs/issues/ISSUE-9-grafana-public-dashboard-url-uses-localhost.md @@ -0,0 +1,139 @@ +# Grafana Public Dashboard URLs Use `localhost` Instead of Real Domain + +**Issue**: [#9](https://github.com/torrust/torrust-tracker-demo/issues/9) +**Related**: [torrust/torrust-tracker-deployer#415](https://github.com/torrust/torrust-tracker-deployer/issues/415) — fix should also be applied upstream in the deployer + +## Problem + +When sharing a Grafana dashboard publicly via **Share → Public dashboard**, Grafana +generates a URL with `localhost:3000` as the base. For example, sharing the +Tracker Overview dashboard produced: + +```text +http://localhost:3000/public-dashboards/186b355b56cd482d9c441a0affdb8ecd +``` + +This URL is not reachable from outside the server. + +## Root Cause + +Grafana builds share URLs using its `root_url` server setting. The default value is +`http://localhost:3000/`. Because `GF_SERVER_ROOT_URL` was not set in the Grafana +container environment, Grafana fell back to this default when constructing the +public dashboard link. + +See the Grafana documentation: +[Externally shared dashboards](https://grafana.com/docs/grafana/next/visualizations/dashboards/share-dashboards-panels/shared-dashboards/) + +## Workaround — Manual URL Reconstruction + +The access token in the generated URL is valid; only the base URL is wrong. The +corrected URL for the Tracker Overview dashboard is: + +```text +https://grafana.torrust-tracker-demo.com/public-dashboards/186b355b56cd482d9c441a0affdb8ecd +``` + +This was verified and confirmed to work. The pattern for fixing any generated URL is: + +| Replace | With | +| ----------------------- | ------------------------------------------ | +| `http://localhost:3000` | `https://grafana.torrust-tracker-demo.com` | + +The remaining two dashboards still need public sharing enabled in the Grafana UI so +their access tokens can be collected: + +| Dashboard | File | UID | Public URL | +| ---------------- | ----------------------------------------------------- | -------------------------- | ------------------------ | +| Tracker Overview | `backups/grafana/dashboards/01-tracker-overview.json` | `torrust-tracker-overview` | _(see workaround above)_ | +| UDP Tracker 1 | `backups/grafana/dashboards/02-udp-tracker-1.json` | `torrust-udp-tracker-1` | _(not yet enabled)_ | +| HTTP Tracker 1 | `backups/grafana/dashboards/03-http-tracker-1.json` | `torrust-http-tracker-1` | _(not yet enabled)_ | + +## Proper Fix + +Add `GF_SERVER_ROOT_URL` to the Grafana service in `docker-compose.yml`, following +the existing environment variable injection pattern — value in `.env`, reference in +`docker-compose.yml`, no hardcoded URLs. + +### Change 1 — `server/opt/torrust/docker-compose.yml` + +```yaml +grafana: + environment: + - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} + - GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL} +``` + +### Change 2 — `server/opt/torrust/.env` + +Add the following entry in the Grafana section: + +```dotenv +# Grafana server root URL — used to generate correct public dashboard share links +GF_SERVER_ROOT_URL='https://grafana.torrust-tracker-demo.com' +``` + +After deploying, restart the Grafana container: + +```sh +docker compose up -d grafana +``` + +New public dashboard share links will automatically use the correct base URL. +Existing access tokens remain valid — only the base URL changes. + +## Additional Work + +As part of this fix, also: + +### 1. Enable public sharing for all three dashboards + +In the Grafana UI, open each dashboard and go to **Share → Public dashboard** to +enable public access and obtain the access token. Collect all three final +public URLs: + +| Dashboard | Expected public URL pattern | +| ---------------- | ----------------------------------------------------------------------------- | +| Tracker Overview | `https://grafana.torrust-tracker-demo.com/public-dashboards/<access-token-1>` | +| UDP Tracker 1 | `https://grafana.torrust-tracker-demo.com/public-dashboards/<access-token-2>` | +| HTTP Tracker 1 | `https://grafana.torrust-tracker-demo.com/public-dashboards/<access-token-3>` | + +### 2. Create a combined dashboard preview image + +Individual screenshots for all three dashboards already exist alongside their JSON +exports in `backups/grafana/dashboards/`: + +| Dashboard | Screenshot | +| ---------------- | ---------------------------------------------------- | +| Tracker Overview | `backups/grafana/dashboards/01-tracker-overview.png` | +| UDP Tracker 1 | `backups/grafana/dashboards/02-udp-tracker-1.png` | +| HTTP Tracker 1 | `backups/grafana/dashboards/03-http-tracker-1.png` | + +Compose those three into a single side-by-side (or grid) image and save it to +`docs/media/grafana-dashboards.webp`. This composite will be the hero image +shown in the README. The individual screenshots in `backups/grafana/dashboards/` +serve as detailed references and can be linked from the README section as well. + +### 3. Update `README.md` + +Add a **Grafana Dashboards** section to the README that shows the composite preview +image, links to all three public dashboards, and links to the individual detailed +screenshots in `backups/grafana/dashboards/`. Suggested placement: after the existing +live demo endpoints block and before the Background section. + +Example structure: + +```markdown +## Grafana Dashboards + +Live public dashboards are available without a Grafana account: + +[![Grafana dashboards preview](docs/media/grafana-dashboards.webp)](https://grafana.torrust-tracker-demo.com) + +| Dashboard | Public link | Screenshot | +| ---------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------------------- | +| Tracker Overview | [open](https://grafana.torrust-tracker-demo.com/public-dashboards/<access-token-1>) | [view](backups/grafana/dashboards/01-tracker-overview.png) | +| UDP Tracker 1 | [open](https://grafana.torrust-tracker-demo.com/public-dashboards/<access-token-2>) | [view](backups/grafana/dashboards/02-udp-tracker-1.png) | +| HTTP Tracker 1 | [open](https://grafana.torrust-tracker-demo.com/public-dashboards/<access-token-3>) | [view](backups/grafana/dashboards/03-http-tracker-1.png) | +``` diff --git a/docs/issues/evidence/ISSUE-19/2026-04-13-baseline-server-snapshot.md b/docs/issues/evidence/ISSUE-19/2026-04-13-baseline-server-snapshot.md new file mode 100644 index 0000000..90fd1ad --- /dev/null +++ b/docs/issues/evidence/ISSUE-19/2026-04-13-baseline-server-snapshot.md @@ -0,0 +1,60 @@ +# ISSUE-19 Evidence: Baseline Server Snapshot (2026-04-13) + +<!-- cspell:ignore estab timewait entr --> + +## Context + +Initial server snapshot collected to establish a baseline before deeper UDP uptime investigation. + +## Command + +```bash +ssh demotracker 'set -e; echo "=== now ==="; date -u; echo "=== uptime ==="; uptime; echo "=== memory ==="; free -h; echo "=== disk ==="; df -h; echo "=== udp sockets summary ==="; ss -u -s; echo "=== docker services ==="; cd /opt/torrust && docker compose ps' +``` + +## Output + +```text +=== now === +Mon Apr 13 14:22:10 UTC 2026 +=== uptime === + 14:22:10 up 8 days, 12:21, 2 users, load average: 6.87, 7.18, 7.11 +=== memory === + total used free shared buff/cache available +Mem: 15Gi 2.5Gi 7.2Gi 5.3Mi 5.8Gi 12Gi +Swap: 0B 0B 0B +=== disk === +Filesystem Size Used Avail Use% Mounted on +tmpfs 1.6G 1.4M 1.6G 1% /run +efivarfs 256K 39K 213K 16% /sys/firmware/efi/efivars +/dev/sda1 150G 9.5G 135G 7% / +tmpfs 7.7G 0 7.7G 0% /dev/shm +tmpfs 5.0M 0 5.0M 0% /run/lock +/dev/sda15 253M 146K 252M 1% /boot/efi +/dev/sdb 49G 2.0G 45G 5% /opt/torrust/storage +tmpfs 1.6G 12K 1.6G 1% /run/user/1000 +=== udp sockets summary === +Total: 206 +TCP: 14462 (estab 4, closed 14445, orphaned 8077, timewait 9) + +Transport Total IP IPv6 +RAW 1 0 1 +UDP 9 6 3 +TCP 17 14 3 +INET 27 20 7 +FRAG 0 0 0 + +Recv-Q Send-Q Local Address:Port Peer Address:PortProcess +=== docker services === +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +caddy caddy:2.10.2 "caddy run --config …" caddy 3 hours ago Up 2 hours (healthy) 0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp, 0.0.0.0:443->443/udp, :::443->443/udp, 2019/tcp +grafana grafana/grafana:12.4.2 "/run.sh" grafana 2 hours ago Up 2 hours (healthy) 3000/tcp +mysql mysql:8.4 "docker-entrypoint.s…" mysql 3 hours ago Up 3 hours (healthy) 3306/tcp, 33060/tcp +prometheus prom/prometheus:v3.5.1 "/bin/prometheus --c…" prometheus 3 hours ago Up 2 hours (healthy) 127.0.0.1:9090->9090/tcp +tracker torrust/tracker:develop "/usr/local/bin/entr…" tracker 3 hours ago Up 3 hours (healthy) 1212/tcp, 0.0.0.0:6868->6868/udp, :::6868->6868/udp, 1313/tcp, 7070/tcp, 0.0.0.0:6969->6969/udp, :::6969->6969/udp +``` + +## Notes + +- This is raw evidence capture only (not a post-mortem). +- Current signal to investigate next: high load average with relatively low memory pressure. diff --git a/docs/issues/evidence/ISSUE-19/2026-04-13-host-kernel-tracker-sanitized-diagnostics.md b/docs/issues/evidence/ISSUE-19/2026-04-13-host-kernel-tracker-sanitized-diagnostics.md new file mode 100644 index 0000000..258f079 --- /dev/null +++ b/docs/issues/evidence/ISSUE-19/2026-04-13-host-kernel-tracker-sanitized-diagnostics.md @@ -0,0 +1,101 @@ +# ISSUE-19 Evidence: Host/Kernel/Tracker Sanitized Diagnostics (2026-04-13) + +<!-- cspell:ignore softirqd kworker ksoftirqd nproc snmp nstat RcvbufErrors SndbufErrors dockerd mysqld loadavg ppid etimes perc containerd multipathd upgr csum rcvbuf sndbuf --> + +## Context + +Second-stage diagnostics snapshot for issue #19. + +This capture intentionally avoids secrets and client-identifying raw payloads: + +- No environment dumps +- No config file dumps +- No raw tracker announce lines with query strings +- Tracker logs are included only as aggregated counts + +## Command + +```bash +ssh demotracker 'set -e; echo "=== now_utc ==="; date -u; echo "=== cpu_load_context ==="; nproc; cat /proc/loadavg; echo "=== top_cpu_processes ==="; ps -eo pid,ppid,comm,%cpu,%mem,etimes --sort=-%cpu | head -n 15; echo "=== top_mem_processes ==="; ps -eo pid,ppid,comm,%mem,%cpu,etimes --sort=-%mem | head -n 15; echo "=== docker_stats_snapshot ==="; docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}\t{{.BlockIO}}"; echo "=== udp_snmp_counters ==="; grep "^Udp:" /proc/net/snmp; echo "=== udp_nstat_counters ==="; nstat -az 2>/dev/null | grep -Ei "Udp(InErrors|NoPorts|RcvbufErrors|SndbufErrors|InDatagrams|OutDatagrams)|Udp6(InErrors|NoPorts|RcvbufErrors|SndbufErrors|InDatagrams|OutDatagrams)" | head -n 50 || true; echo "=== tracker_log_summary_last_60m ==="; cd /opt/torrust; LOGS=$(docker compose logs tracker --since 60m 2>&1 || true); echo "udp_response_error_total=$(printf "%s" "$LOGS" | grep -c "UDP TRACKER: response error" || true)"; echo "udp_warn_abort_no_finished_tasks_total=$(printf "%s" "$LOGS" | grep -c "aborting request: (no finished tasks)" || true)"; echo "invalid_announce_event_total=$(printf "%s" "$LOGS" | grep -c "Invalid announce event" || true)"; echo "cookie_expired_total=$(printf "%s" "$LOGS" | grep -c "cookie value is expired" || true)"; echo "cookie_from_future_total=$(printf "%s" "$LOGS" | grep -c "cookie value is from future" || true)"; echo "invalid_action_total=$(printf "%s" "$LOGS" | grep -c "Invalid action" || true)"' +``` + +## Output + +```text +=== now_utc === +Mon Apr 13 14:31:19 UTC 2026 +=== cpu_load_context === +4 +8.39 7.76 7.41 9/456 909795 +=== top_cpu_processes === + PID PPID COMMAND %CPU %MEM ELAPSED + 789376 789355 caddy 188 2.3 9534 + 761900 761878 torrust-tracker 76.3 3.9 11644 + 909792 909791 bash 50.0 0.0 0 + 1271 1 dockerd 16.2 0.6 736245 + 761878 1 containerd-shim 14.6 0.1 11644 + 761657 761635 mysqld 5.7 3.4 11657 + 909791 909720 sshd 4.7 0.0 0 + 909450 2 kworker/u8:4-ev 2.9 0.0 21 + 898765 2 kworker/u8:0-ev 1.8 0.0 840 + 876355 2 kworker/u8:3-ev 1.4 0.0 2622 + 903431 2 kworker/u8:1-ev 1.2 0.0 487 + 24 2 ksoftirqd/2 1.2 0.0 736255 + 894545 2 kworker/u8:2-ev 1.2 0.0 1167 + 909720 1460563 sshd 1.0 0.0 0 +=== top_mem_processes === + PID PPID COMMAND %MEM %CPU ELAPSED + 761900 761878 torrust-tracker 3.9 76.3 11644 + 761657 761635 mysqld 3.4 5.7 11657 + 789376 789355 caddy 2.3 188 9534 + 789690 789668 grafana 1.7 0.4 9527 + 1271 1 dockerd 0.6 16.2 736245 + 789528 789494 prometheus 0.5 0.0 9533 +1460567 1 systemd-journal 0.4 0.0 287386 + 949 1 containerd 0.3 0.3 736246 + 410 1 multipathd 0.1 0.0 736250 + 762108 1271 docker-proxy 0.1 0.0 11630 + 960 1 unattended-upgr 0.1 0.0 736246 + 762113 1271 docker-proxy 0.1 0.0 11630 + 761878 1 containerd-shim 0.1 14.6 11644 + 789355 1 containerd-shim 0.1 0.0 9534 +=== docker_stats_snapshot === +NAME CPU % MEM USAGE / LIMIT NET I/O BLOCK I/O +grafana 7.54% 95.32MiB / 15.24GiB 548kB / 14.1MB 1.6MB / 19.8MB +prometheus 0.04% 23.5MiB / 15.24GiB 3.15MB / 1.18MB 0B / 4.95MB +caddy 190.76% 382.8MiB / 15.24GiB 40.3GB / 79.9GB 0B / 24.6kB +tracker 73.98% 606.1MiB / 15.24GiB 12.7GB / 10.2GB 4.1kB / 24.6kB +mysql 4.31% 507.7MiB / 15.24GiB 703MB / 815MB 100MB / 4.36GB +=== udp_snmp_counters === +Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti MemErrors +Udp: 367996 147312 18444 431790 18444 0 0 0 0 +=== udp_nstat_counters === +UdpInDatagrams 367996 0.0 +UdpNoPorts 147312 0.0 +UdpInErrors 18444 0.0 +UdpOutDatagrams 431790 0.0 +UdpRcvbufErrors 18444 0.0 +UdpSndbufErrors 0 0.0 +Udp6InDatagrams 81549 0.0 +Udp6NoPorts 39320 0.0 +Udp6InErrors 494 0.0 +Udp6OutDatagrams 9153 0.0 +Udp6RcvbufErrors 494 0.0 +Udp6SndbufErrors 0 0.0 +=== tracker_log_summary_last_60m === +udp_response_error_total=0 +udp_warn_abort_no_finished_tasks_total=1 +invalid_announce_event_total=213 +cookie_expired_total=175 +cookie_from_future_total=18 +invalid_action_total=55 +``` + +## Notes + +- Load is high on a 4 vCPU host (`loadavg` around 8), with `caddy` and + `torrust-tracker` as top CPU consumers. +- UDP receive buffer errors are present (`UdpRcvbufErrors` / `Udp6RcvbufErrors`), + which is a strong signal to investigate socket/kernel buffer pressure. +- Aggregated tracker counters show high invalid/expired/future announce patterns, + but no raw client-identifying request payloads were stored in this file. diff --git a/docs/issues/evidence/ISSUE-19/2026-04-13-progress-and-temporary-conclusions.md b/docs/issues/evidence/ISSUE-19/2026-04-13-progress-and-temporary-conclusions.md new file mode 100644 index 0000000..1e287bf --- /dev/null +++ b/docs/issues/evidence/ISSUE-19/2026-04-13-progress-and-temporary-conclusions.md @@ -0,0 +1,94 @@ +# ISSUE-19 Progress and Temporary Conclusions (2026-04-13) + +<!-- cspell:ignore Rcvbuf rmem netdev --> + +## Scope Reminder + +Goal: explain why newTrackon uptime for +`udp://udp1.torrust-tracker-demo.com:6969/announce` is around 92% instead of +at least 99%, and identify bottlenecks and fixes. + +## What Was Done So Far + +1. Created issue and evidence structure for ongoing investigation. +2. Collected baseline host snapshot (uptime/load/memory/disk/UDP sockets/docker services). +3. Collected sanitized host, kernel, and tracker diagnostics: + process CPU/memory, docker stats, UDP kernel counters, aggregate tracker + error categories. +4. Pulled Prometheus data and validated source mapping between: + - `tracker_stats` (`/api/v1/stats`) + - `tracker_metrics` (`/api/v1/metrics`) +5. Confirmed key metric families and caveats (counter-like vs non-counter). + +## Evidence Files Collected + +- `docs/issues/evidence/ISSUE-19/2026-04-13-baseline-server-snapshot.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-host-kernel-tracker-sanitized-diagnostics.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-3h-summaries.md` +- `docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-source-mapping.md` + +## What We Learned So Far + +1. Both Prometheus jobs are up and provide different data: + - `tracker_stats` includes aggregate UDP series such as `udp6_requests`. + - `tracker_metrics` includes detailed reliability counters such as + `udp_tracker_server_*_total`. +2. Process-level metrics such as `process_cpu_seconds_total` are not exported by + these tracker endpoints, so resource usage must be taken from host/docker/Hetzner. +3. During snapshot windows, host load was high for a 4-vCPU machine, with heavy + CPU usage from `caddy` and `torrust-tracker`. +4. Kernel UDP receive buffer errors were non-zero (`UdpRcvbufErrors`, `Udp6RcvbufErrors`), + indicating potential packet pressure/drops under load. +5. Tracker aggregate log categories show substantial malformed/expired/future + UDP request patterns (not necessarily internal server failure, but real load/error pressure). + +## Temporary Conclusions (Not Final Root Cause) + +1. Nightly restart alone is unlikely to explain uptime as low as ~92%. + Expected impact from a short nightly restart should still be around 99%+. +2. Capacity pressure is currently a strong suspect: + - High sustained load on 4 vCPU + - High tracker and caddy CPU demand + - Non-zero UDP receive buffer errors +3. We still need proof of direct correlation between newTrackon failures and + observed server-side pressure. + +## Potential Actions to Improve Uptime (Based on Current Evidence) + +## Immediate (low risk) + +1. Build a 24-48h minute-resolution dataset for key counters: + `udp_tracker_server_requests_received_total`, + `udp_tracker_server_responses_sent_total{result="error"}`, + `udp_tracker_server_errors_total`, `udp_tracker_server_requests_aborted_total`, + plus host CPU/load and UDP buffer errors. +2. Add synthetic external probes (IPv4 + IPv6) from more than one source + location to compare with newTrackon observations. +3. Capture packet path during degraded windows (`tcpdump`) to verify if probes + arrive and whether replies leave with expected source IP. + +## Short-term tuning + +1. Review and tune UDP socket/kernel buffers (`rmem`/`netdev_max_backlog`) if justified by counters. +2. Reduce avoidable CPU contention in co-located services (especially `caddy` + and `tracker`) and monitor effect. +3. Check connection/request rate limiting behavior and whether malformed traffic + is causing disproportionate processing cost. + +## Capacity/scaling + +1. Run a controlled vertical scaling test (for example 4 vCPU -> 8 vCPU) and + compare error ratios and external probe success rates. +2. If improvement is significant, keep larger sizing or split heavy services + across hosts. + +## Open Questions + +1. Are newTrackon failures concentrated in IPv4, IPv6, or both? +2. Do failure windows align with specific traffic bursts or time-of-day patterns? +3. Is Caddy contributing to resource contention that affects UDP handling indirectly? + +## Next Recommended Step + +Export and store Prometheus `query_range` time-series for the last 24h and 72h +for the selected reliability metrics, then correlate with external probe status. diff --git a/docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-3h-summaries.md b/docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-3h-summaries.md new file mode 100644 index 0000000..b5c63eb --- /dev/null +++ b/docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-3h-summaries.md @@ -0,0 +1,92 @@ +# ISSUE-19 Evidence: Prometheus 3h Summaries (2026-04-13) + +<!-- cspell:ignore promql inet6 iptype memstats urlencode --> + +## Context + +Initial Prometheus pull for UDP and tracker activity over a 3-hour window. + +Note: some metric names queried from `tracker_stats` do not end in `_total` and +Prometheus warns they may not be counters. Treat those results as directional +only, not exact event counts. + +## Command + +```bash +ssh demotracker 'set -e; echo "=== prometheus_up_targets ==="; curl -s "http://127.0.0.1:9090/api/v1/query?query=up"; echo; echo "=== metric_name_sample_tracker_udp ==="; curl -s "http://127.0.0.1:9090/api/v1/label/__name__/values" | tr "," "\n" | grep -Ei "tracker|udp|process_cpu|process_resident|go_memstats|http_tracker|udp_tracker|announce" | head -n 120' + +ssh demotracker 'set -e; echo "=== metric_name_sample_process ==="; curl -s "http://127.0.0.1:9090/api/v1/label/__name__/values" | tr "," "\n" | grep -Ei "process_|go_memstats|go_gc|scrape_duration|scrape_samples" | head -n 120; echo; q(){ expr="$1"; echo "--- $expr"; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=$expr"; echo; }; echo "=== promql_3h_summaries ==="; q "increase(udp4_requests[3h])"; q "increase(udp6_requests[3h])"; q "increase(udp4_errors_handled[3h])"; q "increase(udp6_errors_handled[3h])"; q "increase(udp_requests_aborted[3h])"; q "increase(udp_tracker_server_requests_received_total[3h])"; q "increase(udp_tracker_server_responses_sent_total[3h])"; q "increase(udp_tracker_server_errors_total[3h])"; q "increase(http_tracker_core_requests_received_total[3h])"; q "sum(rate(process_cpu_seconds_total{job=~\"tracker_metrics|tracker_stats\"}[5m])) by (job)"; q "max(process_resident_memory_bytes{job=~\"tracker_metrics|tracker_stats\"}) by (job)"; q "sum(increase(scrape_samples_post_metric_relabeling[3h])) by (job)"' +``` + +## Key Output (excerpt) + +```text +=== prometheus_up_targets === +{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","instance":"tracker:1212","job":"tracker_metrics"},"value":[1776091089.395,"1"]},{"metric":{"__name__":"up","instance":"tracker:1212","job":"tracker_stats"},"value":[1776091089.395,"1"]}]}} + +=== metric_name_sample_tracker_udp === +"udp4_announces_handled" +"udp4_connections_handled" +"udp4_errors_handled" +"udp4_requests" +"udp4_responses" +"udp6_announces_handled" +"udp6_connections_handled" +"udp6_errors_handled" +"udp6_requests" +"udp6_responses" +"udp_requests_aborted" +"udp_tracker_core_requests_received_total" +"udp_tracker_server_connection_id_errors_total" +"udp_tracker_server_errors_total" +"udp_tracker_server_requests_aborted_total" +"udp_tracker_server_requests_accepted_total" +"udp_tracker_server_requests_banned_total" +"udp_tracker_server_requests_received_total" +"udp_tracker_server_responses_sent_total" + +=== promql_3h_summaries === +--- increase(udp6_requests[3h]) +... "value":[1776091152.049,"16720497.711908164"] ... +PromQL info: metric might not be a counter ... "udp6_requests" + +--- increase(udp6_errors_handled[3h]) +... "value":[1776091152.091,"248728.4916400069"] ... +PromQL info: metric might not be a counter ... "udp6_errors_handled" + +--- increase(udp_tracker_server_requests_received_total[3h]) +... port="6969" ... "value":[1776091152.140,"16722828.272908567"] ... + +--- increase(udp_tracker_server_responses_sent_total[3h]) +... request_kind="announce",result="ok",port="6969" ... "value":[1776091152.167,"7712018.825752926"] ... +... request_kind="connect",result="ok",port="6969" ... "value":[1776091152.167,"1940213.299857093"] ... +... request_kind="scrape",result="ok",port="6969" ... "value":[1776091152.167,"41339.018735422665"] ... +... result="error",port="6969" ... "value":[1776091152.167,"248618.45192461362"] ... + +--- increase(udp_tracker_server_errors_total[3h]) +... request_kind="announce",port="6969" ... "value":[1776091152.193,"163041.73026457502"] ... +... request_kind="scrape",port="6969" ... "value":[1776091152.193,"368.21393911740387"] ... +... (unlabeled-kind series),port="6969" ... "value":[1776091152.193,"85208.50772092119"] ... + +--- increase(http_tracker_core_requests_received_total[3h]) +... request_kind="announce",port="7070" ... "value":[1776091152.220,"14897280.595901787"] ... +... request_kind="scrape",port="7070" ... "value":[1776091152.220,"56095.592578095144"] ... + +--- sum(rate(process_cpu_seconds_total{job=~"tracker_metrics|tracker_stats"}[5m])) by (job) +... "result":[] + +--- max(process_resident_memory_bytes{job=~"tracker_metrics|tracker_stats"}) by (job) +... "result":[] +``` + +## Notes + +- Prometheus targets are up for both tracker jobs. +- Useful counter-style metrics are available under + `udp_tracker_server_*_total` and `http_tracker_core_requests_received_total`. +- `process_*` resource metrics are not present in either `tracker_stats` or + `tracker_metrics` (`process_*` queries returned empty results for both jobs). +- Source split confirmed: `udp6_requests` is from `tracker_stats`, while + `udp_tracker_server_*_total` comes from `tracker_metrics`. +- Next step: use `query_range` over 3h for selected `_total` series and export + time series into dedicated files (suitable for graphing and CSV-like analysis). diff --git a/docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-source-mapping.md b/docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-source-mapping.md new file mode 100644 index 0000000..e8e1491 --- /dev/null +++ b/docs/issues/evidence/ISSUE-19/2026-04-13-prometheus-source-mapping.md @@ -0,0 +1,49 @@ +# ISSUE-19 Evidence: Prometheus Source Mapping (2026-04-13) + +<!-- cspell:ignore urlencode --> + +## Context + +Validation of which metrics come from `tracker_stats` vs `tracker_metrics`. + +## Command + +```bash +ssh demotracker 'set -e; q(){ echo "--- $1"; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=$1"; echo; }; q "count({job=\"tracker_stats\"})"; q "count({job=\"tracker_metrics\"})"; q "count(udp6_requests{job=\"tracker_stats\"})"; q "count(udp6_requests{job=\"tracker_metrics\"})"; q "count(udp_tracker_server_requests_received_total{job=\"tracker_metrics\"})"; q "count(udp_tracker_server_requests_received_total{job=\"tracker_stats\"})"; q "count(process_cpu_seconds_total{job=\"tracker_metrics\"})"; q "count(process_cpu_seconds_total{job=\"tracker_stats\"})"' +``` + +## Output + +```text +--- count({job="tracker_stats"}) +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1776091339.778,"33"]}]}} +--- count({job="tracker_metrics"}) +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1776091339.789,"328"]}]}} +--- count(udp6_requests{job="tracker_stats"}) +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1776091339.804,"1"]}]}} +--- count(udp6_requests{job="tracker_metrics"}) +{"status":"success","data":{"resultType":"vector","result":[]}} +--- count(udp_tracker_server_requests_received_total{job="tracker_metrics"}) +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1776091339.841,"2"]}]}} +--- count(udp_tracker_server_requests_received_total{job="tracker_stats"}) +{"status":"success","data":{"resultType":"vector","result":[]}} +--- count(process_cpu_seconds_total{job="tracker_metrics"}) +{"status":"success","data":{"resultType":"vector","result":[]}} +--- count(process_cpu_seconds_total{job="tracker_stats"}) +{"status":"success","data":{"resultType":"vector","result":[]}} +``` + +## Findings + +- `tracker_stats` and `tracker_metrics` are both active and provide different sets. +- `udp6_requests` is present in `tracker_stats` only. +- `udp_tracker_server_requests_received_total` is present in `tracker_metrics` only. +- `process_cpu_seconds_total` is present in neither job (not exported by these tracker endpoints). + +## Implication for Analysis + +Use both sources deliberately: + +- Use `tracker_metrics` for detailed reliability counters (`*_total`, result labels, + server binding labels). +- Use `tracker_stats` for aggregate tracker-level counters/gauges. diff --git a/docs/issues/evidence/ISSUE-19/data/prometheus-2026-04-13/udp6969_requests_received_total.json b/docs/issues/evidence/ISSUE-19/data/prometheus-2026-04-13/udp6969_requests_received_total.json new file mode 100644 index 0000000..bf3bd48 --- /dev/null +++ b/docs/issues/evidence/ISSUE-19/data/prometheus-2026-04-13/udp6969_requests_received_total.json @@ -0,0 +1,746 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "udp_tracker_server_requests_received_total", + "instance": "tracker:1212", + "job": "tracker_metrics", + "server_binding_address_ip_family": "inet6", + "server_binding_address_ip_type": "plain", + "server_binding_ip": "::", + "server_binding_port": "6969", + "server_binding_protocol": "udp" + }, + "values": [ + [ + 1776081443, + "3921790" + ], + [ + 1776081503, + "4013516" + ], + [ + 1776081563, + "4116150" + ], + [ + 1776081623, + "4206995" + ], + [ + 1776081683, + "4298439" + ], + [ + 1776081743, + "4378404" + ], + [ + 1776081803, + "4460962" + ], + [ + 1776081863, + "4564852" + ], + [ + 1776081923, + "4660829" + ], + [ + 1776081983, + "4762157" + ], + [ + 1776082043, + "4839601" + ], + [ + 1776082103, + "4926441" + ], + [ + 1776082163, + "5018166" + ], + [ + 1776082223, + "5106248" + ], + [ + 1776082283, + "5199743" + ], + [ + 1776082343, + "5293348" + ], + [ + 1776082403, + "5381876" + ], + [ + 1776082463, + "5487948" + ], + [ + 1776082523, + "5575085" + ], + [ + 1776082583, + "5672004" + ], + [ + 1776082643, + "5760124" + ], + [ + 1776082703, + "5841950" + ], + [ + 1776082763, + "5939707" + ], + [ + 1776082823, + "6040237" + ], + [ + 1776082883, + "6138464" + ], + [ + 1776082943, + "6238605" + ], + [ + 1776083003, + "6329686" + ], + [ + 1776083063, + "6425461" + ], + [ + 1776083123, + "6521780" + ], + [ + 1776083183, + "6608619" + ], + [ + 1776083243, + "6685980" + ], + [ + 1776083303, + "6767720" + ], + [ + 1776083363, + "6871128" + ], + [ + 1776083423, + "6955984" + ], + [ + 1776083483, + "7044521" + ], + [ + 1776083543, + "7127028" + ], + [ + 1776083603, + "7217169" + ], + [ + 1776083663, + "7317666" + ], + [ + 1776083723, + "7414246" + ], + [ + 1776083783, + "7518261" + ], + [ + 1776083843, + "7614297" + ], + [ + 1776083903, + "7696098" + ], + [ + 1776083963, + "7794249" + ], + [ + 1776084023, + "7892570" + ], + [ + 1776084083, + "7988358" + ], + [ + 1776084143, + "8073257" + ], + [ + 1776084203, + "8158914" + ], + [ + 1776084263, + "8262995" + ], + [ + 1776084323, + "8354615" + ], + [ + 1776084383, + "8449358" + ], + [ + 1776084443, + "8536822" + ], + [ + 1776084503, + "8638633" + ], + [ + 1776084563, + "8737572" + ], + [ + 1776084623, + "8834741" + ], + [ + 1776084683, + "8932337" + ], + [ + 1776084743, + "9013873" + ], + [ + 1776084803, + "9104333" + ], + [ + 1776084863, + "9204493" + ], + [ + 1776084923, + "9291862" + ], + [ + 1776084983, + "9379891" + ], + [ + 1776085043, + "9466088" + ], + [ + 1776085103, + "9555870" + ], + [ + 1776085163, + "9652641" + ], + [ + 1776085223, + "9741919" + ], + [ + 1776085283, + "9827996" + ], + [ + 1776085343, + "9919575" + ], + [ + 1776085403, + "10011344" + ], + [ + 1776085463, + "10107007" + ], + [ + 1776085523, + "10198562" + ], + [ + 1776085583, + "10293760" + ], + [ + 1776085643, + "10375717" + ], + [ + 1776085703, + "10469131" + ], + [ + 1776085763, + "10569821" + ], + [ + 1776085823, + "10668206" + ], + [ + 1776085883, + "10767423" + ], + [ + 1776085943, + "10849136" + ], + [ + 1776086003, + "10948068" + ], + [ + 1776086063, + "11040559" + ], + [ + 1776086123, + "11126751" + ], + [ + 1776086183, + "11220747" + ], + [ + 1776086243, + "11313676" + ], + [ + 1776086303, + "11405405" + ], + [ + 1776086363, + "11503856" + ], + [ + 1776086423, + "11590784" + ], + [ + 1776086483, + "11688869" + ], + [ + 1776086543, + "11778017" + ], + [ + 1776086603, + "11869795" + ], + [ + 1776086663, + "11964206" + ], + [ + 1776086723, + "12055374" + ], + [ + 1776086783, + "12149697" + ], + [ + 1776086843, + "12217625" + ], + [ + 1776086903, + "12293106" + ], + [ + 1776086963, + "12374849" + ], + [ + 1776087023, + "12470173" + ], + [ + 1776087083, + "12557730" + ], + [ + 1776087143, + "12644322" + ], + [ + 1776087203, + "12733393" + ], + [ + 1776087263, + "12832452" + ], + [ + 1776087323, + "12930625" + ], + [ + 1776087383, + "13025752" + ], + [ + 1776087443, + "13121302" + ], + [ + 1776087503, + "13214566" + ], + [ + 1776087563, + "13319620" + ], + [ + 1776087623, + "13414590" + ], + [ + 1776087683, + "13507581" + ], + [ + 1776087743, + "13590786" + ], + [ + 1776087803, + "13685665" + ], + [ + 1776087863, + "13782675" + ], + [ + 1776087923, + "13868505" + ], + [ + 1776087983, + "13955176" + ], + [ + 1776088043, + "14033647" + ], + [ + 1776088103, + "14124743" + ], + [ + 1776088163, + "14228292" + ], + [ + 1776088223, + "14324459" + ], + [ + 1776088283, + "14425297" + ], + [ + 1776088343, + "14517323" + ], + [ + 1776088403, + "14608122" + ], + [ + 1776088463, + "14711875" + ], + [ + 1776088523, + "14799139" + ], + [ + 1776088583, + "14881053" + ], + [ + 1776088643, + "14970200" + ], + [ + 1776088703, + "15056411" + ], + [ + 1776088763, + "15145222" + ], + [ + 1776088823, + "15236342" + ], + [ + 1776088883, + "15335415" + ], + [ + 1776088943, + "15423940" + ], + [ + 1776089003, + "15522592" + ], + [ + 1776089063, + "15621059" + ], + [ + 1776089123, + "15714556" + ], + [ + 1776089183, + "15812718" + ], + [ + 1776089243, + "15902141" + ], + [ + 1776089303, + "15986762" + ], + [ + 1776089363, + "16080356" + ], + [ + 1776089423, + "16175880" + ], + [ + 1776089483, + "16270364" + ], + [ + 1776089543, + "16365194" + ], + [ + 1776089603, + "16463808" + ], + [ + 1776089663, + "16560374" + ], + [ + 1776089723, + "16653404" + ], + [ + 1776089783, + "16758948" + ], + [ + 1776089843, + "16847782" + ], + [ + 1776089903, + "16937348" + ], + [ + 1776089963, + "17032496" + ], + [ + 1776090023, + "17131079" + ], + [ + 1776090083, + "17222729" + ], + [ + 1776090143, + "17303547" + ], + [ + 1776090203, + "17398112" + ], + [ + 1776090263, + "17500815" + ], + [ + 1776090323, + "17591621" + ], + [ + 1776090383, + "17685565" + ], + [ + 1776090443, + "17782180" + ], + [ + 1776090503, + "17878155" + ], + [ + 1776090563, + "17976820" + ], + [ + 1776090623, + "18068205" + ], + [ + 1776090683, + "18168863" + ], + [ + 1776090743, + "18255305" + ], + [ + 1776090803, + "18339548" + ], + [ + 1776090863, + "18423248" + ], + [ + 1776090923, + "18515672" + ], + [ + 1776090983, + "18607737" + ], + [ + 1776091043, + "18691015" + ], + [ + 1776091103, + "18781619" + ], + [ + 1776091163, + "18872738" + ], + [ + 1776091223, + "18964196" + ], + [ + 1776091283, + "19054732" + ], + [ + 1776091343, + "19152992" + ], + [ + 1776091403, + "19247783" + ], + [ + 1776091463, + "19349504" + ], + [ + 1776091523, + "19435963" + ], + [ + 1776091583, + "19528618" + ], + [ + 1776091643, + "19607728" + ], + [ + 1776091703, + "19691639" + ], + [ + 1776091763, + "19778494" + ], + [ + 1776091823, + "19874124" + ], + [ + 1776091883, + "19965360" + ], + [ + 1776091943, + "20046283" + ], + [ + 1776092003, + "20135077" + ], + [ + 1776092063, + "20225245" + ], + [ + 1776092123, + "20314054" + ], + [ + 1776092183, + "20403135" + ], + [ + 1776092243, + "20487315" + ] + ] + } + ] + } +} \ No newline at end of file diff --git a/docs/issues/evidence/ISSUE-21/00-pre-resize-baseline.md b/docs/issues/evidence/ISSUE-21/00-pre-resize-baseline.md new file mode 100644 index 0000000..690cc5b --- /dev/null +++ b/docs/issues/evidence/ISSUE-21/00-pre-resize-baseline.md @@ -0,0 +1,44 @@ +# Pre-Resize Baseline + +<!-- cspell:ignore Rcvbuf --> + +## Context + +Capture baseline immediately before resizing from CCX23 to CCX33. + +## Snapshot + +- Date (UTC): 2026-04-13T15:27:46Z +- Server plan: CCX23 +- vCPU / RAM: 4 / 16 GB +- Traffic allowance: 20 TB + +## Load and Uptime Baseline + +- HTTP1 req/s (Prometheus `rate(...[5m])`): ~1350.05 +- UDP1 req/s (Prometheus `rate(...[5m])`): ~1507.10 +- Total req/s: ~2857.15 +- Req/s per vCPU: ~714.29 +- UDP newTrackon uptime (%): 92.20% + +## Reliability and Capacity Signals + +- `udp_tracker_server_errors_total` (1h/increase): ~52983.82 +- `udp_tracker_server_requests_aborted_total` (1h/increase): ~283.18 +- `udp_tracker_server_responses_sent_total{result="error"}` (1h/increase): ~52983.82 +- Host load average (1m/5m/15m): 6.57 / 6.54 / 6.66 +- UDP receive buffer errors (`UdpRcvbufErrors`, `Udp6RcvbufErrors`): 18444 / 494 + +## Notes + +- Keep command list and links to raw exported artifacts in `data/`. +- Prometheus query method used (`http_rps_5m`): + `sum(rate(http_tracker_core_requests_received_total{server_binding_protocol="http",server_binding_port="7070"}[5m]))` +- Prometheus query method used (`udp_rps_5m`): + `sum(rate(udp_tracker_server_requests_received_total{server_binding_protocol="udp",server_binding_port="6969"}[5m]))` +- Prometheus query method used (`udp_errors_1h`): + `sum(increase(udp_tracker_server_errors_total{server_binding_protocol="udp",server_binding_port="6969"}[1h]))` +- Prometheus query method used (`udp_aborted_1h`): + `sum(increase(udp_tracker_server_requests_aborted_total{server_binding_protocol="udp",server_binding_port="6969"}[1h]))` +- Prometheus query method used (`udp_error_responses_1h`): + `sum(increase(udp_tracker_server_responses_sent_total{server_binding_protocol="udp",server_binding_port="6969",result="error"}[1h]))` diff --git a/docs/issues/evidence/ISSUE-21/01-resize-execution.md b/docs/issues/evidence/ISSUE-21/01-resize-execution.md new file mode 100644 index 0000000..445c154 --- /dev/null +++ b/docs/issues/evidence/ISSUE-21/01-resize-execution.md @@ -0,0 +1,205 @@ +# Resize Execution Log + +<!-- cspell:ignore nproc Rcvbuf perc snmp nstat urlencode poweroff entr --> + +## Planned Change + +- From: CCX23 (4 vCPU, 16 GB RAM, 20 TB) +- To: CCX33 (8 vCPU, 32 GB RAM, 30 TB) +- Expected monthly cost: €62.49/mo + +## Execution Checklist + +- [x] Graceful service shutdown completed via `docker compose down` +- [x] Resize action executed in provider panel +- [x] Server reachable by SSH after resize +- [x] `docker compose ps` healthy +- [x] HTTP endpoint reachable +- [x] UDP endpoint reachable +- [x] Prometheus targets up +- [x] Grafana accessible + +## Pre-Resize Safety Checks + +- [ ] Confirm latest baseline file is complete: + `docs/issues/evidence/ISSUE-21/00-pre-resize-baseline.md` +- [ ] Confirm branch is clean and pushed. +- [ ] Confirm backup window awareness (nightly restart at ~03:00 UTC). +- [ ] Confirm maintenance window and operator availability. + +## Provider Action (Hetzner) + +1. Open server in Hetzner Cloud panel. +2. Resize from **CCX23** to **CCX33**. +3. Wait for resize operation to report complete. +4. Reconnect via SSH and run post-resize checks below. + +## Post-Resize Command Checklist + +Run from local machine: + +```bash +ssh demotracker 'set -e; echo "=== now ==="; date -u; echo "=== cpu_mem ==="; nproc; free -h; echo "=== uptime ==="; uptime; echo "=== docker ==="; cd /opt/torrust && docker compose ps' +``` + +```bash +ssh demotracker 'set -e; cd /opt/torrust; echo "=== docker_stats ==="; docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"' +``` + +```bash +ssh demotracker 'set -e; echo "=== udp_buffers ==="; grep "^Udp:" /proc/net/snmp; nstat -az 2>/dev/null | grep -Ei "UdpRcvbufErrors|Udp6RcvbufErrors" || true' +``` + +```bash +ssh demotracker 'set -e; q(){ expr="$1"; echo "--- $expr"; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=$expr"; echo; }; q "up{job=\"tracker_metrics\"}"; q "up{job=\"tracker_stats\"}"' +``` + +## Endpoint Sanity Checks + +- HTTP tracker health: `curl -fsS "https://http1.torrust-tracker-demo.com/health_check"` +- Tracker API health: `curl -fsS "https://api.torrust-tracker-demo.com/health_check"` +- UDP quick sanity (optional): use existing tracker client tooling and store output under `data/`. + +## Timeline + +- Start (UTC): 2026-04-13T15:36:51Z +- End (UTC): 2026-04-13T15:44:07Z +- Total impact window: ~7m16s (shutdown + provider resize + startup + validation) + +## Pre-Poweroff Graceful Shutdown Log + +Command executed from local machine: + +```bash +ssh demotracker 'set -e; echo "=== resize-prep-start-utc ==="; date -u +%Y-%m-%dT%H:%M:%SZ; cd /opt/torrust; echo "=== docker-compose-ps-before ==="; docker compose ps; echo "=== docker-compose-down ==="; docker compose down; echo "=== docker-compose-ps-after ==="; docker compose ps; echo "=== resize-prep-end-utc ==="; date -u +%Y-%m-%dT%H:%M:%SZ' +``` + +Captured output: + +```text +=== resize-prep-start-utc === +2026-04-13T15:36:51Z +=== docker-compose-ps-before === +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +caddy caddy:2.10.2 "caddy run --config …" caddy 4 hours ago Up 4 hours (healthy) 0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp, 0.0.0.0:443->443/udp, :::443->443/udp, 2019/tcp +grafana grafana/grafana:12.4.2 "/run.sh" grafana 4 hours ago Up 4 hours (healthy) 3000/tcp +mysql mysql:8.4 "docker-entrypoint.s…" mysql 4 hours ago Up 4 hours (healthy) 3306/tcp, 33060/tcp +prometheus prom/prometheus:v3.5.1 "/bin/prometheus --c…" prometheus 4 hours ago Up 4 hours (healthy) 127.0.0.1:9090->9090/tcp +tracker torrust/tracker:develop "/usr/local/bin/entr…" tracker 4 hours ago Up 4 hours (healthy) 1212/tcp, 0.0.0.0:6868->6868/udp, :::6868->6868/udp, 1313/tcp, 7070/tcp, 0.0.0.0:6969->6969/udp, :::6969->6969/udp +=== docker-compose-down === +Container grafana Stopping +Container caddy Stopping +Container grafana Stopped +Container grafana Removing +Container grafana Removed +Container prometheus Stopping +Container prometheus Stopped +Container prometheus Removing +Container prometheus Removed +Container tracker Stopping +Container caddy Stopped +Container caddy Removing +Container caddy Removed +Container tracker Stopped +Container tracker Removing +Container tracker Removed +Container mysql Stopping +Container mysql Stopped +Container mysql Removing +Container mysql Removed +Network torrust_proxy_network Removing +Network torrust_database_network Removing +Network torrust_visualization_network Removing +Network torrust_metrics_network Removing +Network torrust_visualization_network Removed +Network torrust_database_network Removed +Network torrust_metrics_network Removed +Network torrust_proxy_network Removed +=== docker-compose-ps-after === +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +=== resize-prep-end-utc === +2026-04-13T15:37:11Z +``` + +## Immediate Post-Resize Snapshot + +- `nproc`: 8 +- `uptime`: `15:40:20 up 0 min, 1 user, load average: 0.20, 0.06, 0.02` +- `free -h`: `Mem total 30Gi, used 673Mi, available 29Gi` +- `docker compose ps`: all services healthy after startup (caddy, grafana, mysql, + prometheus, tracker) +- `docker stats --no-stream` summary (initial warm-up snapshot): + - `caddy`: high transient CPU during startup (`603.22%`), memory `3.092GiB` + - `tracker`: `153.34%` CPU, memory `364.2MiB` + - `mysql`: `46.54%` CPU, memory `553.2MiB` + - `grafana`: `40.58%` CPU, memory `257.4MiB` + - `prometheus`: `0.06%` CPU, memory `85.14MiB` +- Any regressions observed: + - HTTP1 health endpoint returned `200` with `{"status":"Ok"}`. + - Grafana root returned `302` redirect to `/login` (expected behavior). + - UDP public port probe succeeded on `udp1:6969`. + - API health endpoint returned `500 unauthorized` (same check path appears to + require authorization token; not treated as resize failure). + - Prometheus targets `up{job="tracker_metrics"}` and `up{job="tracker_stats"}` both `1`. + - UDP receive buffer error counters immediately after restart were `0` for both + `UdpRcvbufErrors` and `Udp6RcvbufErrors`. + +## Rollback Criteria (Operational) + +- Server becomes unstable after resize. +- Core services fail to become healthy. +- External endpoints unavailable for prolonged window. + +If rollback is required, document reason and exact time window here. + +## Post-Resize Validation Commands and Key Outputs + +Command (host recovery and internal checks): + +```bash +ssh demotracker 'set -e; echo "=== post-resize-start-utc ==="; date -u +%Y-%m-%dT%H:%M:%SZ; echo "=== host-size-check ==="; nproc; free -h; uptime; echo "=== start-stack ==="; cd /opt/torrust; docker compose up -d; echo "=== docker-compose-ps ==="; docker compose ps; echo "=== docker-stats-no-stream ==="; docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"; echo "=== health-http1 ==="; curl -fsS "https://http1.torrust-tracker-demo.com/health_check"; echo; echo "=== health-api ==="; curl -fsS "https://api.torrust-tracker-demo.com/health_check"; echo; echo "=== prometheus-targets-up ==="; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=up{job=\"tracker_metrics\"}"; echo; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=up{job=\"tracker_stats\"}"; echo; echo "=== udp-buffer-counters ==="; grep "^Udp:" /proc/net/snmp; nstat -az 2>/dev/null | grep -Ei "UdpRcvbufErrors|Udp6RcvbufErrors" || true; echo "=== post-resize-end-utc ==="; date -u +%Y-%m-%dT%H:%M:%SZ' +``` + +Key outputs: + +- `post-resize-start-utc`: `2026-04-13T15:40:20Z` +- `nproc`: `8` +- `free -h` total memory: `30Gi` +- `docker compose ps`: all services `healthy` +- `health-http1`: `200` with `{"status":"Ok"}` +- `health-api`: initial check failed (`502`, then `500 unauthorized`) + +Follow-up command (service stabilization and counters): + +```bash +ssh demotracker 'echo "=== followup-check-utc ==="; date -u +%Y-%m-%dT%H:%M:%SZ; cd /opt/torrust; echo "=== docker-compose-ps ==="; docker compose ps; echo "=== api-health-retries ==="; for i in 1 2 3 4 5; do code=$(curl -s -o /tmp/api_health.out -w "%{http_code}" "https://api.torrust-tracker-demo.com/health_check" || true); echo "try_$i status=$code body=$(cat /tmp/api_health.out 2>/dev/null || true)"; [[ "$code" == "200" ]] && break; sleep 2; done; echo "=== prometheus-target-up ==="; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=up{job=\"tracker_metrics\"}"; echo; curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=up{job=\"tracker_stats\"}"; echo; echo "=== udp-buffer-counters ==="; grep "^Udp:" /proc/net/snmp; nstat -az 2>/dev/null | grep -Ei "UdpRcvbufErrors|Udp6RcvbufErrors" || true' +``` + +Key outputs: + +- `followup-check-utc`: `2026-04-13T15:42:10Z` +- `up{job="tracker_metrics"}`: `1` +- `up{job="tracker_stats"}`: `1` +- `UdpRcvbufErrors`: `0` +- `Udp6RcvbufErrors`: `0` + +External sanity checks: + +```bash +curl -s -o /tmp/http1.out -w "%{http_code}" "https://http1.torrust-tracker-demo.com/health_check" +curl -s -o /tmp/grafana.out -w "%{http_code}" "https://grafana.torrust-tracker-demo.com/" +nc -zvu -w2 udp1.torrust-tracker-demo.com 6969 +``` + +Key outputs: + +- HTTP1 health: `200` +- Grafana root: `302` (`/login` redirect) +- UDP probe: `succeeded` + +## Notes + +- Include exact commands and short outputs (or link to files under `data/`). +- Keep this file chronological and append-only during execution. +- Shutdown duration before poweroff: ~20 seconds. +- User-reported provider resize duration: ~1.5 minutes. diff --git a/docs/issues/evidence/ISSUE-21/02-post-resize-daily-checks.md b/docs/issues/evidence/ISSUE-21/02-post-resize-daily-checks.md new file mode 100644 index 0000000..1ef48a6 --- /dev/null +++ b/docs/issues/evidence/ISSUE-21/02-post-resize-daily-checks.md @@ -0,0 +1,86 @@ +# Post-Resize Daily Checks (7 Days) + +<!-- cspell:ignore Rcvbuf snmp utilization --> + +## Daily Log Template + +| Day | Date (UTC) | HTTP1 req/s | UDP1 req/s | Total req/s | Req/s per vCPU | UDP uptime (%) | UDP errors trend | UDP aborted trend | Host load trend | Notes | +| --- | ---------- | ------------ | ----------- | ----------- | -------------- | -------------- | ---------------- | ----------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| D+1 | 2026-04-20 | ~1564 | ~1015 | ~2579 | ~322 | 83.9% | ~37k/h (pre-fix) | 0 | 6.05/5.49/4.80 | conntrack table full (262144/262144); fixed: nf_conntrack_max→1048576, UDP timeouts reduced; also includes planned resize downtime on 2026-04-14 | +| D+2 | 2026-04-21 | | | | | 85.70% | | | | Rolling uptime still low, but recent [newTrackon raw](https://newtrackon.com/raw) probes are currently successful; likely lag from prior failures | +| D+3 | 2026-04-22 | | | | | | | | | Uptime recovering post-fix; rolling window still catching up | +| D+4 | 2026-04-23 | | | | | | | | | Uptime recovering post-fix; rolling window still catching up | +| D+5 | 2026-04-24 | | | | | | | | | Uptime recovering post-fix; rolling window still catching up | +| D+6 | 2026-04-25 | | | | | | | | | Uptime recovering post-fix; rolling window still catching up | +| D+7 | 2026-04-27 | ~2000 (peak) | ~750 (peak) | | | 99.9% | | | | Target met: 99.9% >= 99.0%; 7-day window complete; issue resolved; peak req/s across 7-day window: HTTP1 ~2000, UDP1 ~750 | + +## D+7 newTrackon Snapshot (2026-04-27) + +Source: newTrackon live tracker table captured 2026-04-27. + +| Tracker URL | Uptime | Status | Checked | +| ----------------------------------------------------- | ------ | ------------------- | -------------- | +| `https://http1.torrust-tracker-demo.com:443/announce` | 99.90% | Working for 2 days | 7 minutes ago | +| `udp://udp1.torrust-tracker-demo.com:6969/announce` | 99.90% | Working for 6 hours | 10 minutes ago | + +Both trackers above the 99.0% target. 7-day observation window complete. +Issue resolved as **Success**. + +## D+7 Live Verification Snapshot (2026-04-27) + +Checked immediately before merging PR #22 to confirm conntrack is healthy at +peak traffic (~750 UDP req/s, ~2000 HTTP req/s). + +Command run: + +```bash +ssh demotracker ' + echo "=== conntrack counts ===" && + sudo sysctl net.netfilter.nf_conntrack_max net.netfilter.nf_conntrack_count && + echo "=== UDP timeouts ===" && + sudo sysctl net.netfilter.nf_conntrack_udp_timeout net.netfilter.nf_conntrack_udp_timeout_stream && + echo "=== dmesg table full ===" && + sudo dmesg -T | grep -i "nf_conntrack: table full" | tail -10 && + echo "(no output = no table-full events)" && + echo "=== UDP receive errors ===" && + cat /proc/net/snmp | grep -E "^Udp:" | + awk "NR==1{for(i=1;i<=NF;i++) h[i]=\$i} NR==2{for(i=1;i<=NF;i++) print h[i]\": \"\$i}" | + grep -E "RcvbufErrors|InErrors|NoPorts" && + echo "=== UDP6 receive errors ===" && + cat /proc/net/snmp6 | grep -E "Udp6RcvbufErrors|Udp6InErrors|Udp6NoPorts" +' +``` + +Results: + +- `nf_conntrack_max`: `1048576` +- `nf_conntrack_count`: `341652` (`32.59%` of max) +- `nf_conntrack_udp_timeout`: `10` +- `nf_conntrack_udp_timeout_stream`: `15` +- `dmesg` table-full events: none +- `UdpRcvbufErrors` (IPv4): `0` +- `UdpInErrors` (IPv4): `0` +- `UdpNoPorts` (IPv4): `57519` — benign; probes to closed ports, not tracker drops +- `Udp6RcvbufErrors` (IPv6): `56` — negligible cumulative counter since boot +- `Udp6InErrors` (IPv6): `56` +- `Udp6NoPorts` (IPv6): `26183` — benign; same as above + +Interpretation: conntrack table is at 32.6% utilization. No table-full events +in dmesg. No IPv4 UDP receive-buffer drops. The 56 IPv6 errors are a cumulative +boot-time counter at ~750 req/s peak and are statistically insignificant. +Conntrack is not overflowing; safe to merge. + +## D+2 Live Verification Snapshot (2026-04-21T07:23:08Z) + +- Host check command source: `ssh demotracker` runtime validation +- `nf_conntrack_max`: `1048576` +- `nf_conntrack_count`: `331258` (`31.59%` of max) +- `nf_conntrack_udp_timeout_stream`: `15` +- `nf_conntrack_udp_timeout`: `10` +- `UdpRcvbufErrors`: `0` +- `Udp6RcvbufErrors`: `0` +- `dmesg` check (`sudo -n dmesg -T | grep -i "nf_conntrack: table full" | tail -10`): no recent matches + +Interpretation: the configured conntrack sizing and UDP timeouts remain active +on the live host, and there is no current evidence of UDP packet drops caused +by conntrack table saturation. diff --git a/docs/issues/evidence/ISSUE-21/03-pre-post-comparison.md b/docs/issues/evidence/ISSUE-21/03-pre-post-comparison.md new file mode 100644 index 0000000..b8d267e --- /dev/null +++ b/docs/issues/evidence/ISSUE-21/03-pre-post-comparison.md @@ -0,0 +1,38 @@ +# Pre vs Post Resize Comparison + +## Goal + +Confirm whether resizing from CCX23 to CCX33 improved UDP uptime to >= 99.0% +and reduced sustained reliability pressure. + +## Summary Table + +| Metric | Pre-resize (CCX23) | Post-resize D+1 (CCX33) | Change | Interpretation | +| --------------------- | ------------------ | ----------------------- | ------- | ----------------------------------------------------------------------------------- | +| HTTP1 req/s | ~1350 | ~1564 | +16% | Traffic grew during observation gap | +| UDP1 req/s | ~1507 | ~1015 | -33% | Traffic lower on D+1; conntrack overflow may have been suppressing visible count | +| Total req/s | ~2857 | ~2579 | -10% | Overall lower on D+1 | +| Req/s per vCPU | ~714 (4 vCPU) | ~322 (8 vCPU) | -55% | Significant headroom gained from resize | +| UDP newTrackon uptime | 92.20% | 83.9% (D+1, pre-fix) | -8.3 pp | Degraded — resize alone was insufficient; conntrack overflow was actual bottleneck | +| UDP errors | ~52984/h | ~37474/h (pre-fix) | -29% | Lower but still high; dropped after conntrack fix applied | +| UDP aborted | ~283/h | 0 | -100% | Gone after resize | +| Host load | 6.57/6.54/6.66 | 6.05/5.49/4.80 | Lower | Load spread over 8 vCPUs vs 4; normalized load dropped from ~1.65 to ~0.76 per vCPU | + +## Decision + +- [x] Success: target met and sustained +- [ ] Partial: improved but below target — resize alone was insufficient; conntrack overflow was the actual bottleneck +- [ ] No improvement: continue with next bottleneck path + +**Status (2026-04-27):** 7-day observation window complete. UDP uptime on newTrackon +reached **99.9%** — above the 99.0% target. The conntrack fix applied on D+1 +(2026-04-20) was the decisive change. The resize from CCX23 → CCX33 was a +necessary supporting step (halved normalized CPU load), but insufficient alone. +Issue resolved. + +## Follow-up Actions + +1. ~~Monitor D+2 through D+7 UDP uptime on newTrackon to confirm fix holds.~~ Done: 99.9% confirmed on 2026-04-27. +2. ~~Verify conntrack fix survives a server reboot (module pre-load + sysctl applied).~~ Done: settings verified live on 2026-04-21. +3. ~~If uptime >= 99.0% by D+7 close issue as resolved.~~ Done: issue resolved. +4. ~~Document in post-mortem if UDP uptime does not recover after fix.~~ N/A: uptime recovered. diff --git a/docs/issues/evidence/ISSUE-21/README.md b/docs/issues/evidence/ISSUE-21/README.md new file mode 100644 index 0000000..1423569 --- /dev/null +++ b/docs/issues/evidence/ISSUE-21/README.md @@ -0,0 +1,17 @@ +# ISSUE-21 Evidence: Server Scale-Up (CCX23 -> CCX33) + +Related issue: [#21](https://github.com/torrust/torrust-tracker-demo/issues/21) + +This folder contains structured evidence for the resize experiment. + +## Files + +- `00-pre-resize-baseline.md` — baseline before resizing +- `01-resize-execution.md` — resize action log and immediate checks +- `02-post-resize-daily-checks.md` — 7-day observation log +- `03-pre-post-comparison.md` — final outcome summary +- `data/` — exported metric artifacts (JSON/CSV) + +## Success Target + +- UDP newTrackon uptime >= 99.0% over rolling 7 days. diff --git a/docs/issues/evidence/ISSUE-29/00-baseline-live-snapshot.md b/docs/issues/evidence/ISSUE-29/00-baseline-live-snapshot.md new file mode 100644 index 0000000..e5ba127 --- /dev/null +++ b/docs/issues/evidence/ISSUE-29/00-baseline-live-snapshot.md @@ -0,0 +1,88 @@ +<!-- cspell:ignore CPUPerc urlencode --> + +# ISSUE-29 Baseline Live Snapshot (Phase 1) + +## Context + +- Issue: [#29](https://github.com/torrust/torrust-tracker-demo/issues/29) +- Capture timestamp (UTC): `2026-05-04T15:20:07Z` +- Goal: record a pre-change baseline before any production tuning action. + +## Commands Used + +```bash +ssh demotracker 'date -u +%Y-%m-%dT%H:%M:%SZ; uptime; nproc' +ssh demotracker 'mpstat -P ALL 1 1' +ssh demotracker 'ps -eo pid,comm,%cpu,%mem,stat --sort=-%cpu | head -20' +ssh demotracker 'cd /opt/torrust && docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"' +ssh demotracker 'sudo sysctl net.netfilter.nf_conntrack_max net.netfilter.nf_conntrack_count net.netfilter.nf_conntrack_udp_timeout net.netfilter.nf_conntrack_udp_timeout_stream' +ssh demotracker 'cat /sys/class/net/eth0/queues/rx-0/rps_cpus; cat /sys/class/net/eth0/queues/rx-0/rps_flow_cnt; sudo sysctl net.core.rps_sock_flow_entries' +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=sum(rate(http_tracker_core_requests_received_total[5m]))"' +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=sum(rate(udp_tracker_server_requests_received_total[5m]))"' +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=sum(rate(http_tracker_core_responses_sent_total{result=\"error\"}[5m]))"' +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=sum(rate(udp_tracker_server_responses_sent_total{result=\"error\"}[5m]))"' +``` + +External uptime sampling source: + +```text +https://newtrackon.com/raw +``` + +## Key Results + +### Host and CPU + +- `uptime`: `load average: 10.46, 9.44, 8.85` +- `nproc`: `8` +- `mpstat` (all CPUs): `%usr=32.81`, `%sys=16.01`, `%soft=20.08`, `%idle=30.97` +- `mpstat` (CPU2): `%soft=100.00`, `%idle=0.00` + +### Top CPU Processes + +- `caddy`: about `279%` +- `torrust-tracker`: about `88.4%` +- `ksoftirqd/2`: about `14.6%` + +### Container CPU Snapshot + +- `caddy`: `323.16%` +- `tracker`: `84.39%` +- `mysql`: `6.63%` +- `grafana`: `0.37%` +- `prometheus`: `0.02%` + +### Conntrack and RX Steering + +- `nf_conntrack_max`: `1048576` +- `nf_conntrack_count`: `424607` +- `nf_conntrack_udp_timeout`: `10` +- `nf_conntrack_udp_timeout_stream`: `15` +- `/sys/class/net/eth0/queues/rx-0/rps_cpus`: `00` +- `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt`: `0` +- `net.core.rps_sock_flow_entries`: `0` + +### Prometheus Request Rates + +- HTTP1 request rate: `1983.1789473684207 req/s` +- UDP1 request rate: `2240.870175438596 req/s` +- Combined request rate: `4224.0491228070167 req/s` +- HTTP error rate query result: empty vector (`0` at sample time) +- UDP error response rate: `26.34035087719298 req/s` + +### newTrackon Raw Snapshot (Sample) + +Observed at capture window from `newtrackon/raw`: + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` + +## Attached Evidence + +- `2026-05-04-htop-snapshot.png` (provided by maintainer) + +## Notes + +- This baseline is intentionally captured before changing any production setting. +- The first planned production experiment remains Phase 2: disable Caddy HTTP/3 + (`443:443/udp`) only, then observe before any further action. diff --git a/docs/issues/evidence/ISSUE-29/01-phase2-disable-http3-execution.md b/docs/issues/evidence/ISSUE-29/01-phase2-disable-http3-execution.md new file mode 100644 index 0000000..e215554 --- /dev/null +++ b/docs/issues/evidence/ISSUE-29/01-phase2-disable-http3-execution.md @@ -0,0 +1,197 @@ +<!-- cspell:ignore CPUPerc --> + +# ISSUE-29 Phase 2 Execution - Disable HTTP/3 UDP Port + +## Context + +- Issue: [#29](https://github.com/torrust/torrust-tracker-demo/issues/29) +- Goal: perform the first isolated production change by removing Caddy UDP 443 + (`443:443/udp`) and restarting only Caddy. +- Change timestamp (UTC): `2026-05-04T15:30:23Z` (edit) and + `2026-05-04T15:30:45Z` (Caddy restart complete). + +## Pre-Change Snapshot + +Pre-change live checks confirmed: + +- `/opt/torrust/docker-compose.yml` contained: + +```yaml +# HTTP/3 (QUIC) +- "443:443/udp" +``` + +- Host listener existed on UDP 443: + +```text +UNCONN 0 0 0.0.0.0:443 0.0.0.0:* +UNCONN 0 0 [::]:443 [::]:* +``` + +- Caddy published ports included UDP 443. + +## Change Applied + +### 1) Repository-side tracked config change + +`server/opt/torrust/docker-compose.yml` was updated to remove the UDP publish +line for Caddy while keeping TCP 80 and 443 unchanged. + +### 2) Live server runtime change + +On `demotracker`, in `/opt/torrust/docker-compose.yml`, the line +`- "443:443/udp"` was removed and only Caddy was recreated: + +```bash +docker compose up -d caddy +``` + +Live diff on server: + +```diff + # HTTPS + - "443:443" + # HTTP/3 (QUIC) +-- "443:443/udp" +``` + +## Immediate Post-Change Validation + +### Service and port checks + +- Caddy container status: `healthy` after recreation. +- Host UDP 443 listener: not present after change. +- HTTP tracker health endpoint: + +```http +GET https://http1.torrust-tracker-demo.com/health_check -> 200 +{"status":"Ok"} +``` + +### API health note (not attributed to this change) + +During immediate checks: + +```http +GET https://api.torrust-tracker-demo.com/health_check -> 500 +Unhandled rejection: Err { reason: "unauthorized" } +``` + +Direct backend check from within Caddy to `tracker:1212/health_check` also +returned HTTP 500 with `unauthorized`, indicating this is upstream API behavior +at capture time, not a reverse-proxy-only failure introduced by the UDP 443 +change. + +### Immediate post-change metrics sample + +Capture timestamp (UTC): `2026-05-04T15:31:33Z` + +- Host load average: `7.63 / 8.49 / 8.67` +- `mpstat` all CPUs: `%usr=33.16`, `%sys=15.79`, `%soft=19.08`, `%idle=31.97` +- `mpstat` CPU2: `%soft=98.02`, `%idle=1.98` +- Container CPU snapshot: + - `caddy`: `319.30%` + - `tracker`: `100.78%` + - `mysql`: `4.66%` + - `grafana`: `0.27%` + - `prometheus`: `0.00%` +- Prometheus rates: + - HTTP1 request rate: `1907.1684210526314 req/s` + - UDP1 request rate: `2269.859649122807 req/s` + +### External probe sample + +From `https://newtrackon.com/raw` during this window: + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` + +## Observation Schedule + +Agreed observation windows for Phase 2: + +| Checkpoint | Target time (UTC) | Status | +| ---------- | -------------------- | -------- | +| T+1 h | 2026-05-04 16:31 | complete | +| T+next day | 2026-05-05 06:16 UTC | complete | + +Capture the same metrics at each checkpoint: `mpstat`, `docker stats`, Prometheus +HTTP1/UDP1 rates, and a `newtrackon.com/raw` sample. + +## T+1 h Observation (2026-05-04T16:54:13Z) + +Capture timestamp (UTC): `2026-05-04T16:54:13Z` (~1 h 36 min after change). + +- Host load average: `8.52 / 8.25 / 8.03` +- `mpstat` all CPUs: `%usr=34.11`, `%sys=15.43`, `%soft=19.58`, `%idle=30.61` +- `mpstat` CPU2: `%soft=100.00`, `%idle=0.00` — **unchanged from pre-change** +- Container CPU snapshot: + - `caddy`: `321.33%` + - `tracker`: `95.47%` + - `mysql`: `7.06%` + - `grafana`: `0.32%` + - `prometheus`: `0.00%` +- `ps` top processes: `caddy 301%`, `torrust-tracker 88.9%`, `ksoftirqd/2 15.0%` +- Prometheus rates: + - HTTP1 request rate: `1834.0 req/s` + - UDP1 request rate: `2440.0 req/s` + +### External probe sample (newtrackon.com/raw) + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` + +### Assessment + +**No improvement observed.** CPU2 remains 100% softirq (`ksoftirqd/2` still +pinned). Load, Caddy CPU (~320%), and tracker CPU (~95%) are all within the same +range as before the change. Removing the Caddy UDP 443 port had no measurable +effect on the softirq saturation, ruling out HTTP/3 (QUIC) as the root cause. + +The Phase 2 change is safe to keep (it was correct hygiene — we have no HTTP/3 +listener anyway), but it did not solve the CPU problem. The investigation must +continue with Phase 3 (RPS/RFS CPU affinity) or a deeper look at why Caddy +alone is consuming ~300% CPU at the observed request rate. + +Keep this single change in place until both checkpoints are completed before +deciding whether to keep HTTP/3 disabled permanently or revert. + +## T+next-day Observation (2026-05-05T06:16:14Z) + +Capture timestamp (UTC): `2026-05-05T06:16:14Z` (~14 h 46 min after change). + +- Host load average: `8.69 / 8.45 / 8.51` +- `mpstat` all CPUs: `%usr=29.57`, `%sys=14.92`, `%soft=19.71`, `%idle=35.67` +- `mpstat` CPU2: `%soft=98.02`, `%idle=1.98` — **unchanged from T+1 h** +- Container CPU snapshot: + - `caddy`: `308.89%` + - `tracker`: `93.22%` + - `mysql`: `7.58%` + - `grafana`: `0.32%` + - `prometheus`: `0.00%` +- `ps` top processes: `caddy 296%`, `torrust-tracker 88.7%`, `ksoftirqd/2 17.8%` +- Prometheus rates: + - HTTP1 request rate: `1909.11 req/s` + - UDP1 request rate: `2178.98 req/s` + +### External probe sample (newtrackon.com/raw) + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` + +### Assessment + +**Confirmed: Phase 2 had no effect.** After ~15 hours all metrics are +statistically identical to both the pre-change baseline and the T+1 h snapshot. +CPU2 remains saturated at ~98% softirq (`ksoftirqd/2` pinned), Caddy is still +~300–310% CPU, and tracker ~88–95%. Load average is stable at ~8.5. + +**Phase 2 decision: keep the change.** Removing `443:443/udp` from Caddy was +correct hygiene (no HTTP/3 listener was in use), but it is not the source of +the CPU issue. The change causes no regressions and removes an unused +port mapping. + +**Root cause is elsewhere.** The softirq saturation on a single CPU is +consistent with all UDP/TCP packet processing being steered to one core. +Confirmed that RPS/RFS are currently disabled on this host. **Phase 3 (enable +RPS/RFS)** is the next isolated change to attempt. diff --git a/docs/issues/evidence/ISSUE-29/02-phase3-enable-rps-rfs-execution.md b/docs/issues/evidence/ISSUE-29/02-phase3-enable-rps-rfs-execution.md new file mode 100644 index 0000000..ad23963 --- /dev/null +++ b/docs/issues/evidence/ISSUE-29/02-phase3-enable-rps-rfs-execution.md @@ -0,0 +1,358 @@ +<!-- cspell:ignore CPUPerc ksoftirqd rps_cpus rps_flow_cnt urlencode --> + +# ISSUE-29 Phase 3 Execution - Enable RPS/RFS + +## Context + +- Issue: [#29](https://github.com/torrust/torrust-tracker-demo/issues/29) +- Goal: perform the second isolated production change by enabling RPS and RFS + to distribute softirq processing across CPUs. +- Trigger: Phase 2 conclusively showed no effect from disabling HTTP/3. + +## Pre-Change Snapshot + +Capture timestamp (UTC): `2026-05-05T06:55:03Z` + +### RX steering state (before) + +Collected at `2026-05-05T06:54:45Z`: + +- `net.core.rps_sock_flow_entries = 0` +- `/sys/class/net/eth0/queues/rx-0/rps_cpus = 00` +- `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt = 0` + +This confirms RPS/RFS were disabled. + +### Performance state (before) + +- Host load average: `9.08 / 9.06 / 8.90` +- `mpstat` all CPUs: `%usr=34.35`, `%sys=15.06`, `%soft=19.68`, `%idle=30.12` +- `mpstat` CPU2: `%soft=100.00`, `%idle=0.00` (fully saturated) +- Container CPU snapshot: + - `caddy`: `323.46%` + - `tracker`: `89.49%` + - `mysql`: `7.06%` + - `grafana`: `0.36%` + - `prometheus`: `0.00%` +- Prometheus rates at pre-change timestamp: + - HTTP1 request rate: `1912.99 req/s` + - UDP1 request rate: `2234.14 req/s` + +## Change Applied + +Apply RPS/RFS live on `demotracker`: + +```bash +sudo sysctl -w net.core.rps_sock_flow_entries=32768 +echo ff | sudo tee /sys/class/net/eth0/queues/rx-0/rps_cpus +echo 4096 | sudo tee /sys/class/net/eth0/queues/rx-0/rps_flow_cnt +``` + +Change timestamp (UTC): `2026-05-05T06:55:19Z` + +### Live verification immediately after change + +- `net.core.rps_sock_flow_entries = 32768` +- `/sys/class/net/eth0/queues/rx-0/rps_cpus = ff` +- `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt = 4096` + +## Immediate Post-Change Validation + +Capture timestamp (UTC): `2026-05-05T06:55:40Z` + +### Performance state (after) + +- Host load average: `9.65 / 9.16 / 8.94` +- `mpstat` all CPUs: `%usr=40.71`, `%sys=14.89`, `%soft=30.15`, `%idle=14.12` +- `mpstat` CPU2: `%soft=48.51`, `%idle=9.90` +- `mpstat` other CPUs: `%soft` now spread across all cores (`24%` to `33%`) +- Container CPU snapshot: + - `caddy`: `411.88%` + - `tracker`: `123.77%` + - `mysql`: `9.67%` + - `grafana`: `0.46%` + - `prometheus`: `0.03%` +- Prometheus rates at post-change timestamp: + - HTTP1 request rate: `1926.15 req/s` + - UDP1 request rate: `2207.48 req/s` + +### External probe sample + +From `https://newtrackon.com/raw` during this window: + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` + +## Assessment + +Initial result is positive for the targeted bottleneck: + +- CPU2 softirq dropped from `100%` to `48.51%`. +- Softirq work is no longer pinned to one core; it is distributed across all 8 + CPUs. +- External tracker health remains `Working` for both HTTP1 and UDP1. + +Interpretation: + +- The one-core softirq hotspot has been mitigated as intended. +- This validates the Phase 3 hypothesis that RX steering imbalance was a major + contributor to the saturation pattern. + +Open questions for follow-up observation: + +- Whether this distribution remains stable over longer windows (T+1 h and + next-day). +- Whether end-to-end CPU headroom improves materially after short-term + settling. + +## Observation Schedule + +Agreed observation windows for Phase 3: + +| Checkpoint | Target time (UTC) | Status | +| ---------- | ----------------- | -------- | +| T+1 h | 2026-05-05 09:13 | complete | +| T+next day | 2026-05-06 09:24 | complete | + +Capture the same metrics at each checkpoint: `mpstat`, `docker stats`, +Prometheus HTTP1/UDP1 rates, and a `newtrackon.com/raw` sample. + +## Operator Handoff Guide (Self-Contained) + +This section is intentionally explicit so another operator can continue Phase 3 +without prior context from chat history. + +### What this investigation is trying to prove + +Phase 3 hypothesis: + +- Enabling RPS/RFS reduces the single-core softirq hotspot (previously CPU2 at + about 100% softirq) by distributing packet processing across CPUs. + +Success condition for follow-up checkpoints: + +- CPU2 `%soft` remains materially below pre-change saturation. +- `%soft` is distributed across multiple CPUs, not pinned to one core. +- HTTP1/UDP1 external status remains `Working`. +- No obvious regression in request rates relative to normal load. + +### Current known-good live settings + +The live host and this repository should both contain: + +- `net.core.rps_sock_flow_entries = 32768` +- `/sys/class/net/eth0/queues/rx-0/rps_cpus = ff` +- `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt = 4096` + +Persistent files expected: + +- Live host: `/etc/sysctl.d/98-rps-rfs.conf`, `/etc/cron.d/rps-rfs` +- Repository mirror: + - `server/etc/sysctl.d/98-rps-rfs.conf` + - `server/etc/cron.d/rps-rfs` + +### Required access and tools + +- SSH alias `demotracker` must work from local machine. +- Prometheus endpoint must be reachable on host at `http://127.0.0.1:9090`. +- `mpstat` must be available on host. + +### Checkpoint command bundle (copy/paste) + +Run these commands from local workstation at each checkpoint. + +1. Capture host/runtime metrics: + +```bash +ssh demotracker 'set -e +echo "=== capture_utc ==="; date -u +%Y-%m-%dT%H:%M:%SZ +echo "=== uptime ==="; uptime +echo "=== rps_state ===" +cat /proc/sys/net/core/rps_sock_flow_entries +cat /sys/class/net/eth0/queues/rx-0/rps_cpus +cat /sys/class/net/eth0/queues/rx-0/rps_flow_cnt +echo "=== mpstat ==="; mpstat -P ALL 1 1 +echo "=== top_cpu ==="; ps -eo pid,comm,%cpu,%mem,stat --sort=-%cpu | head -20 +echo "=== docker_stats ===" +cd /opt/torrust && docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"' +``` + +1. Capture Prometheus rates (numeric values): + +```bash +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=sum(rate(http_tracker_core_requests_received_total[5m]))"' | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["data"]["result"][0]["value"][1])' +ssh demotracker 'curl -sG "http://127.0.0.1:9090/api/v1/query" --data-urlencode "query=sum(rate(udp_tracker_server_requests_received_total[5m]))"' | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["data"]["result"][0]["value"][1])' +``` + +1. Capture external probe sample: + +```bash +curl -s https://newtrackon.com/raw | grep -E 'https://http1.torrust-tracker-demo.com:443/announce|udp://udp1.torrust-tracker-demo.com:6969/announce' | head -10 +``` + +### What to add to this file at each checkpoint + +For each checkpoint (`T+1 h`, `T+next day`), append a new section: + +- `## T+1 h Observation (<timestamp>)` or + `## T+next-day Observation (<timestamp>)` +- Include: + - capture timestamp + - host load average + - `mpstat` all-CPU summary + - `mpstat` CPU2 `%soft` and `%idle` + - whether `%soft` is distributed across CPUs + - container CPU snapshot for `caddy`, `tracker`, `mysql`, `grafana`, `prometheus` + - Prometheus HTTP1 and UDP1 rates + - newtrackon status for HTTP1 and UDP1 +- Add a short assessment paragraph: improved/stable/regressed. + +Then update table status in this file: + +- mark checkpoint `complete` +- replace generic time with actual capture time where appropriate + +### Required updates outside this file + +After each checkpoint, also update issue tracker doc: + +- `docs/issues/ISSUE-29-research-high-cpu-load-after-udp-fix.md` + +Update Phase 3 checklist items as evidence is completed. + +### Commit workflow after each checkpoint + +1. Run validations: + +```bash +./scripts/lint.sh +``` + +1. Commit with signed Conventional Commit, for example: + +```text +docs(issue-29): record phase 3 T+1h observation +``` + +1. Push: + +```bash +git push origin main +``` + +### Guardrails + +- Keep Phase 3 as one variable: do not change unrelated server settings during + observation windows. +- If any live server config is changed, mirror the exact same file content under + `server/` in this repository. +- If metrics look contradictory, capture raw command output first and defer + conclusions until evidence is recorded. + +## T+1h Observation (2026-05-05T09:13:52Z) + +Capture timestamp (UTC): `2026-05-05T09:13:52Z` + +Note: capture was taken at approximately T+2h18m after the Phase 3 change +(`2026-05-05T06:55:19Z`), slightly later than the T+1h target. + +### RPS/RFS state (confirmed active) + +- `net.core.rps_sock_flow_entries = 32768` +- `/sys/class/net/eth0/queues/rx-0/rps_cpus = ff` +- `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt = 4096` + +### Host metrics + +- Host load average: `9.60 / 10.04 / 10.25` +- `mpstat` all CPUs: `%usr=38.68`, `%sys=14.36`, `%soft=32.47`, `%idle=14.36` +- `mpstat` CPU2: `%soft=49.48`, `%idle=8.25` (no longer pinned at 100%) +- `mpstat` other CPUs: `%soft` distributed across all cores (`26%` to `41%`) + +### Container CPU snapshot + +| Container | CPU % | +| ------------ | ------- | +| `caddy` | 455.65% | +| `tracker` | 133.08% | +| `mysql` | 10.16% | +| `grafana` | 0.55% | +| `prometheus` | 0.05% | + +### Prometheus request rates + +- HTTP1 request rate: `1979.53 req/s` +- UDP1 request rate: `2264.42 req/s` + +### External probe sample + +From `https://newtrackon.com/raw` during this window: + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` + +### Assessment + +The T+1h distribution pattern holds stable: + +- CPU2 `%soft` remains at `~49%` (versus `100%` before Phase 3), consistent + with the immediate post-change reading of `48.51%`. +- Softirq work is spread across all 8 CPUs (`26%`–`41%`), with no single-core + saturation. +- Request rates are in the normal operating range (HTTP1 ~1980 req/s, UDP1 + ~2264 req/s), comparable to pre-change levels. +- Both external endpoints remain `Working`. + +No regression detected. The improvement seen at the immediate post-change +snapshot is sustained at T+1h. + +## T+next-day Observation (2026-05-06T09:24:24Z) + +Capture timestamp (UTC): `2026-05-06T09:24:24Z` + +### RPS/RFS state (confirmed active) + +- `net.core.rps_sock_flow_entries = 32768` +- `/sys/class/net/eth0/queues/rx-0/rps_cpus = ff` +- `/sys/class/net/eth0/queues/rx-0/rps_flow_cnt = 4096` + +### Host metrics + +- Host load average: `11.83 / 11.59 / 10.82` +- `mpstat` all CPUs: `%usr=34.65`, `%sys=15.41`, `%soft=31.08`, `%idle=18.73` +- `mpstat` CPU2: `%soft=49.49`, `%idle=11.11` (still far below pre-change 100%) +- `mpstat` other CPUs: `%soft` distributed across all cores (`26.80%` to `33.33%`) + +### Container CPU snapshot + +| Container | CPU % | +| ------------ | ------- | +| `caddy` | 436.17% | +| `tracker` | 117.37% | +| `mysql` | 10.02% | +| `grafana` | 4.01% | +| `prometheus` | 0.01% | + +### Prometheus request rates + +- HTTP1 request rate: `1937.70 req/s` +- UDP1 request rate: `2208.68 req/s` + +### External probe sample + +From `https://newtrackon.com/raw` during this window: + +- `https://http1.torrust-tracker-demo.com:443/announce` -> `Working` (`09:25:49 UTC`) +- `udp://udp1.torrust-tracker-demo.com:6969/announce` -> `Working` (`09:24:22 UTC`) + +### Assessment + +The T+next-day checkpoint confirms the same pattern seen at T+1h: + +- CPU2 `%soft` remains stable at `~49.5%` (not re-pinned to 100%). +- Softirq work stays distributed across all 8 CPUs. +- Both external endpoints remain `Working`. + +However, total host load remains high (`11.83` on an 8-vCPU machine), which +confirms that while RPS/RFS mitigated the single-core hotspot, overall capacity +headroom is still limited at current traffic levels. diff --git a/docs/issues/evidence/ISSUE-29/03-draft-blog-post-rps-rfs-softirq-hotspot.md b/docs/issues/evidence/ISSUE-29/03-draft-blog-post-rps-rfs-softirq-hotspot.md new file mode 100644 index 0000000..8df2991 --- /dev/null +++ b/docs/issues/evidence/ISSUE-29/03-draft-blog-post-rps-rfs-softirq-hotspot.md @@ -0,0 +1,342 @@ +<!-- cspell:ignore ksoftirqd softirq rps rfs cpus cpupct mpstat Prometheus QUIC newtrackon conntrack udp urlencode --> + +# Draft Blog Post: How We Fixed a One-Core Packet Processing Bottleneck in Torrust Tracker + +> Draft status: work in progress from ISSUE-29. +> +> This draft is written for developers. Linux networking terms are explained as we +> use them. + +## TL;DR + +We found that one CPU core was saturated doing kernel packet work (`softirq`), +while other cores still had capacity. This caused persistent high load and +limited headroom. + +We tested two isolated changes: + +1. Disable HTTP/3 (QUIC) on Caddy by removing UDP 443. +2. Enable RPS/RFS to distribute packet processing across CPUs. + +Final result: + +- Disabling HTTP/3 did **not** improve the bottleneck. +- Enabling RPS/RFS immediately reduced CPU2 softirq from `100%` to `48.51%` + and spread packet work across all 8 CPUs. +- At T+next-day, distribution remained stable (`CPU2 %soft=49.49%`), and both + public endpoints were still `Working`. +- However, global host load remained high (`11.83 / 11.59 / 10.82`), so this + fix improved balance but did not create enough long-term capacity headroom. + +## What Problem We Detected + +We were seeing sustained high CPU pressure on a production tracker host. + +Key symptoms: + +- `mpstat` showed one core (CPU2) pinned by kernel soft interrupts. +- `ksoftirqd/2` kept showing up near the top CPU consumers. +- User-space processes (`caddy`, `torrust-tracker`) were busy, but the unusual + pattern was the **single-core softirq hotspot**. + +For developers: what is `softirq`? + +- When packets arrive at the NIC, part of the work happens in kernel networking + paths, not in your app process. +- Linux accounts part of this work as `softirq`. +- If this work stays concentrated on one core, that core can saturate even when + other cores are available. + +This was exactly our pattern. + +## How We Analyzed the Problem and Found the Cause + +### Method: one variable at a time + +We deliberately avoided changing many things at once. + +- Phase 2 changed only HTTP/3 exposure on Caddy. +- Phase 3 changed only RPS/RFS steering. + +This lets us attribute effects to a specific change. + +## Phase 2: Hypothesis "HTTP/3 is causing the bottleneck" + +### Why this was a reasonable hypothesis + +Caddy had UDP 443 configured for HTTP/3 (QUIC). If QUIC traffic or handling was +causing extra kernel pressure, removing it could reduce softirq load. + +### Change applied + +In `server/opt/torrust/docker-compose.yml`, we removed UDP 443 from Caddy: + +```diff + ports: + - "80:80" + - "443:443" +- - "443:443/udp" +``` + +Then we restarted only Caddy on the live server: + +```bash +docker compose up -d caddy +``` + +### Commands we used and what they mean + +**1. `mpstat -P ALL 1 1`** + +- Shows per-CPU usage split (`%usr`, `%sys`, `%soft`, `%idle`, etc.). +- We use this to detect whether packet work is concentrated on one CPU. + +**2. `ps -eo pid,comm,%cpu,%mem,stat --sort=-%cpu | head -20`** + +- Shows top CPU-consuming processes. +- We use this to check `ksoftirqd/<N>` and main containers. + +**3. `docker stats --no-stream`** + +- One-shot CPU/memory snapshot per container. +- We use this to compare `caddy` and `tracker` before/after changes. + +**4. Prometheus HTTP/UDP rate queries:** + +```bash +curl -sG "http://127.0.0.1:9090/api/v1/query" \ + --data-urlencode "query=sum(rate(http_tracker_core_requests_received_total[5m]))" + +curl -sG "http://127.0.0.1:9090/api/v1/query" \ + --data-urlencode "query=sum(rate(udp_tracker_server_requests_received_total[5m]))" +``` + +- We use these to ensure traffic level is comparable across snapshots. + +**5. External service check from `newtrackon.com/raw`** + +- Confirms public tracker endpoints still behave as expected. + +### Phase 2 outputs (selected) + +Immediate and follow-up snapshots stayed in the same range: + +- CPU2 `%soft`: `98-100%` (still saturated) +- Caddy CPU: about `309-321%` +- Tracker CPU: about `93-100%` +- HTTP1/UDP1 externally: `Working` + +Example checkpoint output summary: + +```text +T+next-day (2026-05-05T06:16:14Z) +CPU2 %soft=98.02, %idle=1.98 +caddy=308.89%, tracker=93.22% +HTTP1 rate=1909.11 req/s, UDP1 rate=2178.98 req/s +``` + +### Phase 2 conclusion + +Disabling HTTP/3 (QUIC) was good hygiene (removed unused UDP publish), but it +was **not** the cause of the one-core softirq bottleneck. + +## Phase 3: Hypothesis "packet processing is not being distributed" + +### Why this hypothesis fit the evidence + +If all incoming packet handling is effectively funneled to one CPU, that CPU can +hit softirq saturation first. + +RPS/RFS are kernel mechanisms that help distribute receive-side packet handling: + +- RPS (Receive Packet Steering): allows packet processing to be steered to + multiple CPUs. +- RFS (Receive Flow Steering): improves steering for flow-to-CPU locality. + +### Pre-change state (important) + +Before Phase 3, steering features were disabled: + +```text +net.core.rps_sock_flow_entries = 0 +/sys/class/net/eth0/queues/rx-0/rps_cpus = 00 +/sys/class/net/eth0/queues/rx-0/rps_flow_cnt = 0 +``` + +Interpretation: + +- `00` means no CPU mask configured for RPS. +- `0` flow values mean RFS is effectively off. + +### Change applied (live) + +```bash +sudo sysctl -w net.core.rps_sock_flow_entries=32768 +echo ff | sudo tee /sys/class/net/eth0/queues/rx-0/rps_cpus +echo 4096 | sudo tee /sys/class/net/eth0/queues/rx-0/rps_flow_cnt +``` + +What each line does: + +**1. `rps_sock_flow_entries=32768`** + +- Creates a global flow table used by RFS. + +**2. `rps_cpus=ff`** + +- CPU bitmask `ff` (`0xff`) enables CPUs 0-7 as packet-processing targets. + +**3. `rps_flow_cnt=4096`** + +- Per-RX-queue flow count used for RFS decisions. + +### Immediate post-change output (selected) + +```text +Post-change (2026-05-05T06:55:40Z) +all CPUs: %soft=30.15 +CPU2: %soft=48.51, %idle=9.90 +other CPUs: %soft distributed across ~24-33% +``` + +Container snapshot around same time: + +```text +caddy=411.88% +tracker=123.77% +mysql=9.67% +``` + +Traffic remained comparable: + +```text +pre HTTP=1912.99 req/s, UDP=2234.14 req/s +post HTTP=1926.15 req/s, UDP=2207.48 req/s +``` + +External endpoint sample remained healthy: + +```text +https://http1.torrust-tracker-demo.com:443/announce -> Working +udp://udp1.torrust-tracker-demo.com:6969/announce -> Working +``` + +### Phase 3 conclusion after T+1h and T+next-day checkpoints + +The specific bottleneck we targeted improved and stayed improved: + +- Immediate: CPU2 stopped being hard-pinned at 100% softirq. +- T+1h: CPU2 stayed around `49.48%`, with distributed softirq across all cores. +- T+next-day: CPU2 stayed around `49.49%`, still distributed. + +This is strong evidence that receive-side steering imbalance was a major cause. + +But there is an important second-order learning: + +- Fixing a one-core packet-processing hotspot does not automatically reduce + total system load enough for comfortable operation. +- Even with better CPU distribution, the host can remain near saturation under + current traffic. + +## What We Changed in Configuration Files + +This section is explicit so readers can reproduce the patch. + +### 1) Caddy compose change (Phase 2 hygiene) + +File: `server/opt/torrust/docker-compose.yml` + +Relevant final section: + +```yaml +ports: + - "80:80" + - "443:443" + # HTTP/3 (QUIC) intentionally disabled during ISSUE-29 Phase 2 experiment +``` + +Change made: + +```diff +- - "443:443/udp" +``` + +### 2) Persistent kernel setting for RFS + +File: `server/etc/sysctl.d/98-rps-rfs.conf` + +Final file content: + +```conf +# RPS/RFS tuning to distribute NIC RX softirq work across CPUs. +# See: docs/udp-conntrack-runbook.md and ISSUE-29. + +# Global socket flow table size used by RFS. +net.core.rps_sock_flow_entries = 32768 +``` + +### 3) Persistent boot-time sysfs writes for RPS/RFS + +File: `server/etc/cron.d/rps-rfs` + +Final file content: + +```cron +# Persist RPS/RFS sysfs settings across reboot. +# See: docs/udp-conntrack-runbook.md and ISSUE-29. + +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +@reboot root echo ff > /sys/class/net/eth0/queues/rx-0/rps_cpus && echo 4096 > /sys/class/net/eth0/queues/rx-0/rps_flow_cnt +``` + +## Why We Use Both Live Changes and Repo Mirror + +This repository mirrors deployed server config under `server/`. + +Operational rule we followed: + +- If a live config file changes, the same final file content must be committed + under `server/...`. + +This keeps production reproducible and reviewable. + +## Final Operational Decision and Follow-Up + +Decisions taken after completing the full observation window: + +1. Keep RPS/RFS enabled permanently on the current host. +2. Close ISSUE-29 as completed for the tuning scope. +3. Track capacity follow-up in ISSUE-30 (server scale-up planning). + +Interpretation: + +- ISSUE-29 succeeded technically for its target (remove single-core softirq + saturation). +- ISSUE-29 did not eliminate overall capacity risk at current traffic. +- The next lever is capacity (scale-up), not another immediate packet-path + tuning change. + +## Suggested Blog Structure (for torrust.com/blog) + +1. Problem statement and user-visible symptoms. +2. Why one-core softirq saturation matters for UDP/TCP tracker traffic. +3. Phase 2 experiment (HTTP/3) and why it was ruled out. +4. Phase 3 experiment (RPS/RFS) and immediate improvements. +5. Exact patch files and deployment commands. +6. Follow-up checkpoints and final production decision. + +## References + +- Issue plan: `docs/issues/ISSUE-29-research-high-cpu-load-after-udp-fix.md` +- Follow-up scaling issue: `docs/issues/ISSUE-30-scale-up-server-for-sustained-load.md` +- Phase 2 evidence: `docs/issues/evidence/ISSUE-29/01-phase2-disable-http3-execution.md` +- Phase 3 evidence: `docs/issues/evidence/ISSUE-29/02-phase3-enable-rps-rfs-execution.md` +- Htop snapshots: + - `docs/issues/evidence/ISSUE-29/2026-05-04-htop-snapshot.png` + - `docs/issues/evidence/ISSUE-29/2026-05-05-htop-snapshot.png` + - `docs/issues/evidence/ISSUE-29/2026-05-06-htop-snapshot.png` +- Background runbook: `docs/udp-conntrack-runbook.md` +- Previous related blog post: + - https://torrust.com/blog/nf-conntrack-overflow-docker-udp-tracker diff --git a/docs/issues/evidence/ISSUE-29/2026-05-04-htop-snapshot.png b/docs/issues/evidence/ISSUE-29/2026-05-04-htop-snapshot.png new file mode 100644 index 0000000..eadb0f7 Binary files /dev/null and b/docs/issues/evidence/ISSUE-29/2026-05-04-htop-snapshot.png differ diff --git a/docs/issues/evidence/ISSUE-29/2026-05-05-htop-snapshot.png b/docs/issues/evidence/ISSUE-29/2026-05-05-htop-snapshot.png new file mode 100644 index 0000000..5f56629 Binary files /dev/null and b/docs/issues/evidence/ISSUE-29/2026-05-05-htop-snapshot.png differ diff --git a/docs/issues/evidence/ISSUE-29/2026-05-06-htop-snapshot.png b/docs/issues/evidence/ISSUE-29/2026-05-06-htop-snapshot.png new file mode 100644 index 0000000..90c0b97 Binary files /dev/null and b/docs/issues/evidence/ISSUE-29/2026-05-06-htop-snapshot.png differ diff --git a/docs/issues/evidence/ISSUE-31/00-immediate-post-change-snapshot.md b/docs/issues/evidence/ISSUE-31/00-immediate-post-change-snapshot.md new file mode 100644 index 0000000..3f83a8e --- /dev/null +++ b/docs/issues/evidence/ISSUE-31/00-immediate-post-change-snapshot.md @@ -0,0 +1,238 @@ +# ISSUE-31 Evidence — Immediate Post-Change Snapshot + +**Issue:** [#31 Re-enable Caddy HTTP/3 and Document ISSUE-29 Rationale][issue-31] +**Change applied:** Re-add `443:443/udp` port mapping to Caddy in docker-compose.yml +**Captured:** 2026-05-07 ~13:17 UTC +**Author:** Jose Celano + +[issue-31]: https://github.com/torrust/torrust-tracker-demo/issues/31 + +--- + +## 1. Pre-Change State + +Since ISSUE-29 Phase 2 (2026-05-04), the Caddy service in +`/opt/torrust/docker-compose.yml` on the live server had the `443:443/udp` +port removed. The stated reason was "remove unused port binding", but +no measurable CPU benefit was observed. See +`docs/issues/ISSUE-29-research-high-cpu-load-after-udp-fix.md` for the +original decision text, now marked as superseded by ISSUE-31. + +--- + +## 2. Change Applied + +### Local repo change (`server/opt/torrust/docker-compose.yml`) + +```yaml +ports: + - "80:80" + - "443:443" + # HTTP/3 (QUIC) re-enabled in ISSUE-31 + - "443:443/udp" +``` + +### Live server patch (via `sed` over SSH) + +```bash +# Insert the UDP port line after the TCP 443 line on the live server +ssh demotracker "sed -i '/\"443:443\"$/a \ # HTTP\/3 (QUIC) re-enabled in ISSUE-31\n - \"443:443\/udp\"' \ + /opt/torrust/docker-compose.yml" + +# Recreate only the Caddy container (zero downtime for other services) +ssh demotracker "cd /opt/torrust && docker compose up -d caddy" +``` + +Expected output: + +```text +Container caddy Recreated +Container caddy Started +``` + +> **Cosmetic note:** The `sed` command inserted the new block correctly but +> left an orphan comment `# HTTP/3 (QUIC)` that was already present below +> the `443:443` line. The live server ports block therefore has one redundant +> comment line. This is harmless and will be cleaned up in the next +> docker-compose sync. + +--- + +## 3. Post-Change Validations + +### 3.1 Container ports + +```bash +ssh demotracker "docker ps --format 'table {{.Names}}\t{{.Ports}}' | grep caddy" +``` + +Output: + +```text +caddy 0.0.0.0:80->80/tcp, :::80->80/tcp, + 0.0.0.0:443->443/tcp, :::443->443/tcp, + 0.0.0.0:443->443/udp, :::443->443/udp, + 2019/tcp +``` + +Both `0.0.0.0:443->443/udp` and `:::443->443/udp` are present. ✅ + +### 3.2 Host UDP 443 listener (kernel socket) + +```bash +ssh demotracker "ss -ulnp | grep ':443'" +``` + +Output: + +```text +UNCONN 0 0 0.0.0.0:443 0.0.0.0:* +UNCONN 0 0 [::]:443 [::]:* +``` + +IPv4 and IPv6 UDP sockets are bound. ✅ + +### 3.3 Caddy container health + +```bash +ssh demotracker "docker inspect --format='{{.State.Health.Status}}' caddy" +``` + +Output: `healthy` ✅ + +### 3.4 HTTP tracker health + +```bash +ssh demotracker \ + "curl -s -o /dev/null -w '%{http_code}' https://http1.torrust-tracker-demo.com:443/health_check" +``` + +Output: `200` (body: `{"status":"Ok"}`) ✅ + +### 3.5 API health (expected 500 — pre-existing) + +```bash +ssh demotracker \ + "curl -s -o /dev/null -w '%{http_code}' https://http1.torrust-tracker-demo.com/api/v1/stats" +``` + +Output: `500` (unauthorized — pre-existing, not caused by this change) ⚠️ pre-existing + +--- + +## 4. Metrics Snapshot (T+0 ~1 min post-restart) + +> **Important caveat:** These metrics were captured approximately 1 minute +> after the Caddy container restart. Several values are anomalously high and +> are almost certainly transient artifacts of the container initialization, +> not a steady-state signal. Do not use T+0 values for rollback evaluation. +> Use T+1h and T+next-day checkpoints instead. + +### 4.1 Host load average + +```bash +ssh demotracker "uptime" +``` + +Output (approximate): + +```text + 13:17:09 up ... load average: 18.93, 13.89, 11.95 +``` + +Comment: elevated 1-min average (18.93) consistent with a freshly restarted +container; 5-min and 15-min averages trail higher from ISSUE-29 baseline but +this single-point reading is not meaningful. + +### 4.2 CPU usage snapshot (`mpstat -P ALL 1 1`) + +| Metric | Value | +| ------ | ----- | +| %usr | 3.25 | +| %sys | 90.88 | +| %soft | 5.75 | +| %idle | 0.12 | + +High `%sys` during container restart is expected. Softirq load was distributed +across all CPUs (2–13% each), confirming RPS/RFS is still operating correctly +(no single-core bottleneck). + +### 4.3 Docker container resource usage (T+0) + +| Container | CPU% | MEM | +| ---------- | ------ | --------- | +| caddy | 717% | 763.2 MiB | +| tracker | 58.73% | 826.5 MiB | +| mysql | 6.27% | — | +| grafana | 2.91% | — | +| prometheus | 0.03% | — | + +Caddy at 717% is a well-known container-restart spike. No action required. + +### 4.4 HTTP and UDP request rates (Prometheus) + +Queries executed via Prometheus API: + +```bash +# HTTP1 tracker +ssh demotracker \ + "curl -sG 'http://localhost:9090/api/v1/query' \ + --data-urlencode 'query=sum(rate(http_tracker_core_requests_received_total[5m]))'" + +# UDP1 tracker +ssh demotracker \ + "curl -sG 'http://localhost:9090/api/v1/query' \ + --data-urlencode 'query=sum(rate(udp_tracker_server_requests_received_total[5m]))'" +``` + +| Metric | Value (req/s) | +| -------------- | ------------- | +| HTTP1 req rate | 2116.38 | +| UDP1 req rate | 2238.84 | + +Both values are in normal operating range. ✅ + +--- + +## 5. Observation Schedule + +| Checkpoint | Target Time (UTC) | Status | File | +| ---------- | ----------------- | ------- | -------------------------------------- | +| T+0 (this) | 2026-05-07 ~13:17 | ✅ Done | `00-immediate-post-change-snapshot.md` | +| T+1h | 2026-05-07 ~14:17 | ✅ Done | `01-t1h-snapshot.md` | +| T+next-day | 2026-05-08 ~13:17 | ✅ Done | `02-next-day-snapshot.md` | + +--- + +## 6. Rollback Triggers + +From ISSUE-31 implementation plan: + +1. Caddy sustained CPU > baseline × 1.20 over a 24 h window +2. Host load average sustained > baseline × 1.15 over a 24 h window +3. HTTP1 or UDP1 availability regression reported on newtrackon.com + +T+0 snapshot is not sufficient to evaluate any of these in isolation. Follow-up +checkpoints were captured in `01-t1h-snapshot.md` and +`02-next-day-snapshot.md`. + +Current status after later checkpoints: + +- External availability regression was not observed. +- Strict sustained-24h CPU/load evaluation became inconclusive because the host + restarted between checkpoints, interrupting the continuous 24h comparison + window. + +--- + +## 7. Summary + +- UDP 443 successfully re-enabled at the Caddy edge proxy on both IPv4 and IPv6. +- Caddy is healthy and all downstream services (HTTP tracker, MySQL, Grafana, + Prometheus) are unaffected. +- HTTP1 and UDP1 tracker request rates are in normal operating range. +- T+0 CPU/load figures are transient restart artifacts — not a signal. +- T+1h and T+next-day checkpoints were later recorded and confirmed stable edge + HTTP/3 operation. +- No rollback action was indicated by the later checkpoints, although strict + sustained-24h CPU/load comparison was interrupted by a host restart. diff --git a/docs/issues/evidence/ISSUE-31/01-t1h-snapshot.md b/docs/issues/evidence/ISSUE-31/01-t1h-snapshot.md new file mode 100644 index 0000000..0793f5b --- /dev/null +++ b/docs/issues/evidence/ISSUE-31/01-t1h-snapshot.md @@ -0,0 +1,228 @@ +# ISSUE-31 Evidence — T+1h Snapshot + +**Issue:** [#31 Re-enable Caddy HTTP/3 and Document ISSUE-29 Rationale][issue-31] +**Scope:** Post-change stability checkpoint (T+1h target window) +**Captured:** 2026-05-07 16:29:55 UTC +**Author:** Jose Celano + +[issue-31]: https://github.com/torrust/torrust-tracker-demo/issues/31 + +--- + +## 1. Context + +The T+1h checkpoint was scheduled for approximately 14:17 UTC. This capture was +performed at 16:29 UTC, still valid as a post-change steady-state checkpoint. + +Primary objective: verify that re-enabling Caddy edge HTTP/3 (`443:443/udp`) +remains healthy and does not show immediate regression versus the restart-biased +T+0 snapshot. + +--- + +## 2. Commands Executed and Outputs + +### 2.1 Full capture command + +```bash +cat <<'EOS' | ssh demotracker 'bash -s' +set -euo pipefail + +echo '=== ISSUE-31 T+1h snapshot ===' +date -u +"UTC_TIME=%Y-%m-%dT%H:%M:%SZ" + +echo + +echo '--- caddy ports ---' +docker ps --format 'table {{.Names}}\t{{.Ports}}' | grep caddy || true + +echo + +echo '--- udp listeners :443 ---' +ss -ulnp | grep ':443' || true + +echo + +echo '--- caddy health ---' +docker inspect --format='{{.State.Health.Status}}' caddy + +echo + +echo '--- http tracker health ---' +code=$(curl -s -o /tmp/http1_hc.out -w '%{http_code}' https://http1.torrust-tracker-demo.com:443/health_check || true) +echo "HTTP_CODE=$code" +head -c 400 /tmp/http1_hc.out || true +echo + +echo + +echo '--- api stats endpoint status (expected pre-existing unauthorized) ---' +code=$(curl -s -o /tmp/api_stats.out -w '%{http_code}' https://http1.torrust-tracker-demo.com/api/v1/stats || true) +echo "HTTP_CODE=$code" +head -c 400 /tmp/api_stats.out || true +echo + +echo + +echo '--- host uptime/load ---' +uptime + +echo + +echo '--- mpstat -P ALL 1 1 ---' +mpstat -P ALL 1 1 + +echo + +echo '--- docker stats --no-stream ---' +docker stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}' + +echo + +echo '--- prometheus HTTP1 rate (5m) ---' +curl -sG 'http://localhost:9090/api/v1/query' \ + --data-urlencode 'query=sum(rate(http_tracker_core_requests_received_total[5m]))' + +echo + +echo '--- prometheus UDP1 rate (5m) ---' +curl -sG 'http://localhost:9090/api/v1/query' \ + --data-urlencode 'query=sum(rate(udp_tracker_server_requests_received_total[5m]))' + +echo + +echo '--- newtrackon raw page sample ---' +curl -sL 'https://newtrackon.com/raw' | head -c 800 + +echo +EOS +``` + +### 2.2 Key outputs captured + +Timestamp: + +```text +UTC_TIME=2026-05-07T16:29:55Z +``` + +Caddy ports: + +```text +caddy 0.0.0.0:80->80/tcp, [::]:80->80/tcp, + 0.0.0.0:443->443/tcp, [::]:443->443/tcp, + 0.0.0.0:443->443/udp, [::]:443->443/udp, 2019/tcp +``` + +UDP 443 listeners: + +```text +UNCONN 0 0 0.0.0.0:443 0.0.0.0:* +UNCONN 0 0 [::]:443 [::]:* +``` + +Caddy health: + +```text +healthy +``` + +HTTP tracker health: + +```text +HTTP_CODE=200 +{"status":"Ok"} +``` + +API stats endpoint: + +```text +HTTP_CODE=404 +``` + +Host load: + +```text +load average: 11.46, 9.87, 9.73 +``` + +`mpstat -P ALL 1 1` (aggregate): + +| Metric | Value | +| ------ | ----- | +| %usr | 34.69 | +| %sys | 14.54 | +| %soft | 30.23 | +| %idle | 20.41 | + +`docker stats --no-stream`: + +| Container | CPU% | Memory | +| ---------- | ------- | ------------------ | +| caddy | 420.91% | 481.9MiB / 30.6GiB | +| tracker | 120.47% | 795.3MiB / 30.6GiB | +| mysql | 5.94% | 663.3MiB / 30.6GiB | +| grafana | 0.48% | 309.1MiB / 30.6GiB | +| prometheus | 0.00% | 99.62MiB / 30.6GiB | + +Prometheus rates (5m): + +```text +HTTP1: 1902.150877192982 req/s +UDP1: 2053.5017543859644 req/s +``` + +newtrackon raw sample: + +```text +Returns HTML document content (same behavior as T+0), not a plain tracker status line. +``` + +--- + +## 3. Comparison vs T+0 Snapshot + +| Signal | T+0 (13:17) | T+1h checkpoint (16:29) | Notes | +| ------------------ | ----------- | ----------------------- | ------------------------ | +| Caddy health | healthy | healthy | Stable | +| UDP 443 published | yes | yes | Stable | +| UDP 443 listeners | yes | yes | Stable | +| HTTP health code | 200 | 200 | Stable | +| HTTP1 rate (req/s) | 2116.38 | 1902.15 | Normal variation | +| UDP1 rate (req/s) | 2238.84 | 2053.50 | Normal variation | +| Caddy CPU% | 717.12 | 420.91 | Lower than restart spike | +| Load avg (1m) | 18.93 | 11.46 | Lower than restart spike | + +--- + +## 4. Rollback Trigger Check (Interim) + +Configured rollback triggers require sustained 24h regressions. + +1. Caddy CPU > baseline x 1.20 sustained 24h: **not evaluable yet (window incomplete)** +2. Host load > baseline x 1.15 sustained 24h: **not evaluable yet (window incomplete)** +3. External availability regression on HTTP1/UDP1: **not observed in this checkpoint** + +Interim conclusion: no rollback action indicated at T+1h checkpoint. + +--- + +## 5. Notes and Open Point + +The API stats endpoint returned `404` in this snapshot, while the earlier note +recorded `500 unauthorized`. + +This endpoint behavior is not part of ISSUE-31 acceptance criteria and does not +affect the HTTP/3 edge re-enable validation. It should be tracked separately if +API stats exposure is required for operations. + +--- + +## 6. Summary + +- Caddy edge HTTP/3 publish mapping remains active and healthy on IPv4 and IPv6. +- Request rates remain in expected operating range. +- T+0 restart artifacts have subsided (lower load and lower Caddy CPU than the + immediate post-restart spike). +- No rollback trigger is met at this checkpoint. +- Next required checkpoint: T+next-day snapshot. diff --git a/docs/issues/evidence/ISSUE-31/02-next-day-snapshot.md b/docs/issues/evidence/ISSUE-31/02-next-day-snapshot.md new file mode 100644 index 0000000..91ca8b3 --- /dev/null +++ b/docs/issues/evidence/ISSUE-31/02-next-day-snapshot.md @@ -0,0 +1,239 @@ +# ISSUE-31 Evidence — T+next-day Snapshot + +**Issue:** [#31 Re-enable Caddy HTTP/3 and Document ISSUE-29 Rationale][issue-31] +**Scope:** Post-change next-day checkpoint (T+next-day) +**Captured:** 2026-05-08 08:37:47 UTC +**Author:** Jose Celano + +[issue-31]: https://github.com/torrust/torrust-tracker-demo/issues/31 + +--- + +## 1. Context + +This snapshot is the next-day checkpoint for ISSUE-31 after re-enabling Caddy +edge HTTP/3 (`443:443/udp`). + +Goal: confirm that the edge HTTP/3 mapping remains active and healthy, and +continue rollback-trigger evaluation after the T+0 and T+1h checkpoints. + +--- + +## 2. Commands Executed and Outputs + +### 2.1 Full capture command + +```bash +cat <<'EOS' | ssh demotracker 'bash -s' +set -euo pipefail + +echo '=== ISSUE-31 T+next-day snapshot ===' +date -u +"UTC_TIME=%Y-%m-%dT%H:%M:%SZ" + +echo + +echo '--- caddy ports ---' +docker ps --format 'table {{.Names}}\t{{.Ports}}' | grep caddy || true + +echo + +echo '--- udp listeners :443 ---' +ss -ulnp | grep ':443' || true + +echo + +echo '--- caddy health ---' +docker inspect --format='{{.State.Health.Status}}' caddy + +echo + +echo '--- http tracker health ---' +code=$(curl -s -o /tmp/http1_hc.out -w '%{http_code}' https://http1.torrust-tracker-demo.com:443/health_check || true) +echo "HTTP_CODE=$code" +head -c 400 /tmp/http1_hc.out || true +echo + +echo + +echo '--- api stats endpoint status ---' +code=$(curl -s -o /tmp/api_stats.out -w '%{http_code}' https://http1.torrust-tracker-demo.com/api/v1/stats || true) +echo "HTTP_CODE=$code" +head -c 400 /tmp/api_stats.out || true +echo + +echo + +echo '--- host uptime/load ---' +uptime + +echo + +echo '--- mpstat -P ALL 1 1 ---' +mpstat -P ALL 1 1 + +echo + +echo '--- docker stats --no-stream ---' +docker stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}' + +echo + +echo '--- prometheus HTTP1 rate (5m) ---' +curl -sG 'http://localhost:9090/api/v1/query' \ + --data-urlencode 'query=sum(rate(http_tracker_core_requests_received_total[5m]))' + +echo + +echo '--- prometheus UDP1 rate (5m) ---' +curl -sG 'http://localhost:9090/api/v1/query' \ + --data-urlencode 'query=sum(rate(udp_tracker_server_requests_received_total[5m]))' + +echo + +echo '--- newtrackon raw page sample ---' +curl -sL 'https://newtrackon.com/raw' | head -c 800 + +echo +EOS +``` + +### 2.2 Key outputs captured + +Timestamp: + +```text +UTC_TIME=2026-05-08T08:37:47Z +``` + +Caddy ports: + +```text +caddy 0.0.0.0:80->80/tcp, [::]:80->80/tcp, + 0.0.0.0:443->443/tcp, [::]:443->443/tcp, + 0.0.0.0:443->443/udp, [::]:443->443/udp, 2019/tcp +``` + +UDP 443 listeners: + +```text +UNCONN 0 0 0.0.0.0:443 0.0.0.0:* +UNCONN 0 0 [::]:443 [::]:* +``` + +Caddy health: + +```text +healthy +``` + +HTTP tracker health: + +```text +HTTP_CODE=200 +{"status":"Ok"} +``` + +API stats endpoint: + +```text +HTTP_CODE=404 +``` + +Host load: + +```text +load average: 7.38, 7.15, 7.42 +``` + +`mpstat -P ALL 1 1` (aggregate): + +| Metric | Value | +| ------ | ----- | +| %usr | 31.66 | +| %sys | 14.93 | +| %soft | 28.70 | +| %idle | 24.71 | + +`docker stats --no-stream`: + +| Container | CPU% | Memory | +| ---------- | ------- | ------------------ | +| caddy | 396.15% | 529.2MiB / 30.6GiB | +| tracker | 156.71% | 601MiB / 30.6GiB | +| mysql | 4.26% | 611.8MiB / 30.6GiB | +| grafana | 0.53% | 300.1MiB / 30.6GiB | +| prometheus | 0.09% | 98.15MiB / 30.6GiB | + +Prometheus rates (5m): + +```text +HTTP1: 1872.2701754385962 req/s +UDP1: 1924.9859649122807 req/s +``` + +newtrackon raw sample: + +```text +Returns HTML document content, not a plain tracker status line. +``` + +--- + +## 3. Comparison vs Prior Checkpoints + +| Signal | T+0 (13:17) | T+1h (16:29) | T+next-day (08:37) | Notes | +| ------------------ | ------------ | ------------ | ------------------ | ------------------------- | +| Caddy health | healthy | healthy | healthy | Stable | +| UDP 443 published | yes | yes | yes | Stable | +| UDP 443 listeners | yes | yes | yes | Stable | +| HTTP health code | 200 | 200 | 200 | Stable | +| API stats code | 500/unauth\* | 404 | 404 | Out of scope for ISSUE-31 | +| HTTP1 rate (req/s) | 2116.38 | 1902.15 | 1872.27 | Normal variation | +| UDP1 rate (req/s) | 2238.84 | 2053.50 | 1924.99 | Normal variation | +| Caddy CPU% | 717.12 | 420.91 | 396.15 | Lower than restart spike | +| Load avg (1m) | 18.93 | 11.46 | 7.38 | Lower than restart spike | + +\*The T+0 note recorded `500 unauthorized`; from T+1h onward, `/api/v1/stats` returned `404`. + +--- + +## 4. Rollback Trigger Check (Next-day) + +Configured rollback triggers: + +1. Caddy CPU > baseline x 1.20 sustained 24h +2. Host load > baseline x 1.15 sustained 24h +3. External availability regression on HTTP1/UDP1 + +### Assessment + +- Trigger 3: **not observed** in this checkpoint. +- Triggers 1 and 2: **inconclusive for sustained 24h comparison** because host uptime + was only ~6h at capture time and kernel changed to `6.8.0-111-generic`, indicating + a host restart and new observation window. + +Interim conclusion: no immediate rollback signal; continue observation from this +new host-uptime window if strict sustained-24h criteria are required. + +--- + +## 5. Notable Environment Change + +This snapshot shows: + +- `uptime`: `up 6:37` +- `uname` (from `mpstat` header): `Linux 6.8.0-111-generic` + +Compared to earlier snapshots (`6.8.0-110-generic`), this indicates host reboot +or kernel update occurred between checkpoints. This interrupts strict 24h +continuity for baseline comparisons. + +--- + +## 6. Summary + +- Caddy edge HTTP/3 mapping (`443:443/udp`) remains active and healthy on IPv4 and IPv6. +- HTTP and UDP tracker traffic rates remain in expected operating range. +- No availability regression observed. +- Sustained-24h rollback-trigger evaluation for CPU/load is interrupted by host reboot; + results are operationally healthy but not a strict continuous 24h baseline window. diff --git a/docs/issues/evidence/README.md b/docs/issues/evidence/README.md new file mode 100644 index 0000000..d598efc --- /dev/null +++ b/docs/issues/evidence/README.md @@ -0,0 +1,27 @@ +# Issue Investigation Evidence + +This folder stores raw investigation artifacts for issue work in progress. + +Use this for command transcripts, log extracts, metrics snapshots, and network +captures collected before a root cause is confirmed. + +Do not use this folder for final post-mortems. Post-mortems belong in +`docs/post-mortems/` after the incident is understood. + +## Layout + +- `docs/issues/evidence/ISSUE-<N>/` + +## Suggested File Naming + +- `YYYY-MM-DD-baseline-<topic>.md` +- `YYYY-MM-DD-logs-<service>.md` +- `YYYY-MM-DD-network-<topic>.md` +- `YYYY-MM-DD-metrics-<topic>.md` + +Keep each capture in a separate file with: + +1. Context +2. Exact command(s) +3. Raw output +4. Short notes/hypothesis diff --git a/docs/linting.md b/docs/linting.md new file mode 100644 index 0000000..667ef83 --- /dev/null +++ b/docs/linting.md @@ -0,0 +1,208 @@ +# Linting + +This document describes how linting works in this repository, how to install +the required tools, and how to troubleshoot common failures. + +## Canonical Command + +Run all linters with: + +```sh +./scripts/lint.sh +``` + +This is the preferred command for local validation before committing or opening +a pull request. + +## Pre-commit Script + +The repository also provides a pre-commit wrapper: + +```sh +./scripts/pre-commit.sh +``` + +For now this script delegates directly to `./scripts/lint.sh`. + +## Git Pre-commit Hook + +A tracked Git hook is included at `.githooks/pre-commit` and runs +`./scripts/pre-commit.sh` automatically before `git commit`. + +Enable the tracked hooks once per local clone: + +```sh +git config core.hooksPath .githooks +``` + +After that, `git commit` will execute the repository's pre-commit checks +automatically. + +The script runs these linters in order: + +```sh +linter markdown +linter yaml +linter cspell +linter shellcheck +``` + +It stops on the first failure. + +## What the Script Checks + +- Markdown using `.markdownlint.json` +- YAML using `.yamllint-ci.yml` +- Spelling using `cspell.json` and `project-words.txt` +- Shell scripts using ShellCheck + +## Installation + +### Required tools + +Install the Rust wrapper used by the script: + +```sh +cargo install torrust-linting --locked +``` + +Install system tools: + +```sh +sudo apt-get update +sudo apt-get install -y yamllint shellcheck +``` + +### Node-based tools + +The `linter` command may try to install Node-based tools automatically, but in +practice it is more reliable to install them explicitly in a user-writable npm +prefix. + +Example setup: + +```sh +export NPM_CONFIG_PREFIX="$HOME/.local" +export PATH="$HOME/.local/bin:$PATH" +npm install -g markdownlint-cli cspell@8.17.5 +``` + +If you want those settings permanently, add the two `export` lines to your +shell profile. + +## Recommended Local Workflow + +Run this before committing: + +```sh +export NPM_CONFIG_PREFIX="$HOME/.local" +export PATH="$HOME/.local/bin:$PATH" +./scripts/pre-commit.sh +``` + +If the script passes, the repository is clean for Markdown, YAML, spelling, and +shell script linting. + +## Running a Single Linter + +You can run individual linters through the same wrapper: + +```sh +linter markdown +linter yaml +linter cspell +linter shellcheck +``` + +This is useful when fixing one class of failures at a time. + +## Troubleshooting + +### `linter: command not found` + +The Rust wrapper is not installed. + +Fix: + +```sh +cargo install torrust-linting --locked +``` + +### npm `EACCES` while installing markdownlint or cspell + +The wrapper may try to install npm packages globally under `/usr/local`, which +fails for non-root users. + +Fix: use a user-local npm prefix. + +```sh +export NPM_CONFIG_PREFIX="$HOME/.local" +export PATH="$HOME/.local/bin:$PATH" +``` + +Then rerun `./scripts/lint.sh`. + +### `Unsupported NodeJS version` from cspell + +At the time this document was written, the latest `cspell` release required +Node.js `>=22.18.0`, while this repository workflow was being run with Node 20. + +Fix options: + +- Upgrade Node.js to a compatible version. +- Or install a Node 20-compatible cspell version explicitly: + +```sh +npm install -g cspell@8.17.5 +``` + +### `yamllint` triggers a sudo prompt during `./scripts/lint.sh` + +The wrapper may try to install `yamllint` itself. + +Preferred fix: install it manually ahead of time. + +```sh +sudo apt-get install -y yamllint +``` + +### `shellcheck` is missing + +Install it explicitly: + +```sh +sudo apt-get install -y shellcheck +``` + +### Spell checker reports a valid technical term + +Add the term to `project-words.txt`, one word per line, then rerun the linter. + +Examples already added during recent documentation work include: + +- `Mailgun` +- `tulpn` + +### Markdown lint fails on indentation or tabs + +Markdown linting is strict about formatting details such as tab characters. +Use spaces for wrapped list items and paragraph continuations. + +## Files Involved + +- [scripts/lint.sh](../scripts/lint.sh) +- [scripts/pre-commit.sh](../scripts/pre-commit.sh) +- [.githooks/pre-commit](../.githooks/pre-commit) +- [project-words.txt](../project-words.txt) +- [cspell.json](../cspell.json) +- [.markdownlint.json](../.markdownlint.json) +- [.yamllint-ci.yml](../.yamllint-ci.yml) + +## Notes + +- `./scripts/lint.sh` is the canonical repository command. +- `./scripts/pre-commit.sh` is the hook-friendly entry point for commit-time checks. +- If you document new product names, services, or command tokens, update + `project-words.txt` as part of the same change. +- Run the full script again after fixing targeted issues to ensure the complete + repository still passes. diff --git a/docs/media/grafana-dashboards.webp b/docs/media/grafana-dashboards.webp new file mode 100644 index 0000000..8ec2137 Binary files /dev/null and b/docs/media/grafana-dashboards.webp differ diff --git a/docs/media/newtrackon-trackers.png b/docs/media/newtrackon-trackers.png new file mode 100644 index 0000000..36e1a1f Binary files /dev/null and b/docs/media/newtrackon-trackers.png differ diff --git a/docs/security/reviews/2026-04/01-caddy-and-https.md b/docs/security/reviews/2026-04/01-caddy-and-https.md new file mode 100644 index 0000000..11d3560 --- /dev/null +++ b/docs/security/reviews/2026-04/01-caddy-and-https.md @@ -0,0 +1,95 @@ +# Caddy and HTTPS - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- Public HTTPS entry points +- Virtual host routing +- Reverse proxy behavior +- Header trust boundaries +- Unexpected backend exposure + +## Hypotheses + +- Only the intended virtual hosts are publicly routed by Caddy. +- No default or catch-all route exposes an unintended backend. +- Public HTTPS entry points expose the tracker API and Grafana login surface, + not only public dashboards. + +## Evidence Reviewed + +- [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- Live responses from `api.torrust-tracker-demo.com`, + `grafana.torrust-tracker-demo.com`, and `http1.torrust-tracker-demo.com` +- Focused path probes on the three public HTTP hosts + +## Checks Performed + +- Confirmed configured virtual hosts: `api.torrust-tracker-demo.com`, + `http1.torrust-tracker-demo.com`, `http2.torrust-tracker-demo.com`, and + `grafana.torrust-tracker-demo.com`. +- Confirmed Caddy proxies the tracker API to `tracker:1212` and Grafana to + `grafana:3000`. +- Confirmed the committed compose config intentionally publishes port `80` for + Caddy as `HTTP (ACME HTTP-01 challenge)`. +- Confirmed no catch-all site block is present in the committed Caddyfile. +- Confirmed live responses: + - API root returns `HTTP/2 500` + - API `/api/health_check` returns `HTTP/2 200` + - Grafana root returns `HTTP/2 302` with `Location: /login` + - HTTP tracker root returns `HTTP/2 404` + - HTTP tracker `/announce` returns `HTTP/2 200` +- Confirmed transport and header behavior on the three main public hosts: + - Plain `http://` requests are redirected to `https://` with `HTTP 308` + - HTTPS responses include `via: 1.1 Caddy` + - API and HTTP tracker responses include `x-request-id` + - Grafana responses include Grafana's own hardening headers such as + `x-content-type-options: nosniff` and `x-frame-options: deny` + - No `Strict-Transport-Security` header was observed on the tested host root + responses +- Confirmed additional live route behavior: + - Protected API v1 paths returned `HTTP 500` without a token + - Grafana `/login` returns `HTTP 200` + - Grafana `/api/health` returns `HTTP 200` + - HTTP tracker `/health_check` returns `HTTP 200` + +## Findings or Non-Findings + +- Confirmed finding recorded: the public HTTPS hosts redirect plaintext traffic + to HTTPS but do not advertise HSTS, which leaves first-visit clients exposed + to downgrade or SSL-stripping risk on hostile networks. +- Non-finding: port `80` is intentionally exposed for automatic certificate + issuance and renewal via ACME HTTP-01, so the current HSTS finding is about + missing browser-side HTTPS pinning rather than the mere existence of an HTTP + listener. +- No Caddy routing misconfiguration is confirmed yet. Current evidence supports + the configured host mapping and expected route separation. +- No evidence yet of an unexpected extra virtual host or cross-host route leak. +- No plaintext service exposure was observed on the tested public hosts; all + tested `http://` requests were redirected to HTTPS. + +## Open Questions + +- Does the live Caddy runtime differ from the committed Caddyfile? +- Which Grafana auth modes are enabled behind the public login page? +- Is the API-host behavior for `/` and similar unrelated paths generated by the + tracker router, by Caddy path handling, or by a deployed revision mismatch? +- Is the public Grafana health endpoint an intentional exposure or just a side + effect of exposing the full Grafana UI? +- Could the deployment move to another ACME challenge mode later, or is HTTP-01 + the intended long-term certificate path for this setup? + +## Next Actions + +- Validate live host and path behavior from the public endpoints. +- Review the tracker API source to understand which routes are reachable through + the public API host. +- Check whether any unexpected paths on the public virtual hosts expose extra + endpoints or debugging behavior. +- Decide whether HSTS should be added directly in Caddy or through the + deployer-generated template path. +- Continue checking path behavior only as needed; current evidence already + suggests the main next step is source-backed API analysis. diff --git a/docs/security/reviews/2026-04/02-tracker-api.md b/docs/security/reviews/2026-04/02-tracker-api.md new file mode 100644 index 0000000..08ea1a8 --- /dev/null +++ b/docs/security/reviews/2026-04/02-tracker-api.md @@ -0,0 +1,116 @@ +# Tracker API - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- Public API routes +- Authentication and token handling +- Sensitive or administrative operations +- Error handling and information disclosure + +## Hypotheses + +- The public API host may expose privileged endpoints guarded only by a static + admin token. +- The deployed tracker image may contain debug or diagnostics endpoints not + obvious from the repository config. + +## Evidence Reviewed + +- [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [../../../server/opt/torrust/storage/tracker/etc/tracker.toml](../../../server/opt/torrust/storage/tracker/etc/tracker.toml) +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- Live responses from `https://api.torrust-tracker-demo.com/`, + `https://api.torrust-tracker-demo.com/api/health_check`, and + `https://api.torrust-tracker-demo.com/api/v1/*` +- Live responses from additional tested API paths such as `/api`, `/stats`, + `/metrics`, `/swagger`, and `/openapi.json` +- Upstream tracker source for `packages/axum-rest-tracker-api-server` + +## Checks Performed + +- Confirmed the public API hostname proxies directly to `tracker:1212`. +- Confirmed the committed Caddy config does not rewrite or strip path prefixes on + the API hostname. +- Confirmed an admin access token is injected through the environment variable + `TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN`. +- Confirmed repository config alone does not identify the full public route + surface; source review was required to distinguish `/api/health_check` from + the versioned API routes. +- Confirmed live unauthenticated `GET /api/health_check` returns `HTTP/2 200` + with the expected health payload. +- Confirmed live unauthenticated requests to `/`, `/health_check`, and `/api` + return `HTTP/2 500`. +- Confirmed the API root response body exposes internal error text: + `Unhandled rejection: Err { reason: "unauthorized" }`. +- Confirmed unauthenticated versioned API routes also return the same `HTTP 500` + behavior, including `/api/v1/stats`, `/api/v1/torrents`, + `/api/v1/whitelist`, and `/api/v1/keys`. +- Confirmed clearly unmatched paths also return auth-shaped `HTTP 500` + responses, including `/api/v1/notfound`, `/api/v1/foo/bar`, + `/api/notfound`, `/swagger`, and `/`. +- Confirmed the same `HTTP 500` behavior across additional tested API-host + paths: `/login`, `/api`, `/api/`, `/stats`, `/metrics`, `/health_check`, + `/announce`, `/swagger`, `/openapi.json`, and `/robots.txt`. +- Confirmed path-shape behavior around the public health route: + - Exact `GET /api/health_check` returns `HTTP/2 200` + - `GET /api/health_check/` returns `HTTP/2 500` with the same auth-shaped + body as the protected routes + - `GET /api/health_check/foo` also returns the same auth-shaped `HTTP/2 500` +- Confirmed `/swagger` also returns the same body text: + `Unhandled rejection: Err { reason: "unauthorized" }`. +- Confirmed `OPTIONS /` also returns `HTTP/2 500`, which suggests the problem is + not limited to one specific GET route. +- Confirmed upstream source defines a public `GET /api/health_check` handler + outside the auth middleware. +- Confirmed upstream source applies the v1 auth middleware at router level and + maps missing tokens, invalid tokens, and malformed auth headers to + `unhandled_rejection_response(...)`, which returns `500` with internal error + text such as `unauthorized`. +- Confirmed the current upstream router is built by adding the versioned API + routes first, wrapping that router with the auth middleware, and only then + adding the exact public `/api/health_check` route. +- Confirmed the upstream auth middleware returns early on missing or invalid + auth data instead of always calling `next.run(request).await`, which explains + why requests that land inside the auth-wrapped router can fail with the same + auth-shaped `500` before any downstream `404` is produced. +- Confirmed the live deployment exposes all three internal auth failure strings + on `/api/v1/stats`: + - No token: `Unhandled rejection: Err { reason: "unauthorized" }` + - Invalid bearer token: `Unhandled rejection: Err { reason: "token not valid" }` + - Unsupported auth scheme: `Unhandled rejection: Err { reason: "unknown token provided" }` +- Confirmed invalid bearer tokens also change the response bodies on clearly + unmatched paths such as `/api/v1/notfound`, `/api/notfound`, `/swagger`, and + `/`, which is consistent with the current auth middleware returning before a + downstream `404` can be generated. + +## Findings or Non-Findings + +- Confirmed finding recorded: unauthenticated access to the versioned tracker + API host currently returns `500` and exposes internal error text instead of a + proper client error, including on apparently unrelated or unmatched paths. +- Non-finding: the dedicated public API health endpoint at `/api/health_check` + is reachable and returns `200`, which is consistent with the upstream route + table. + +## Open Questions + +- Which exact tracker revision backs the deployed `torrust/tracker:develop` + image? +- Exactly which request set is captured by the auth-wrapped router versus the + standalone public health route in the deployed build? +- Should the public API hostname expose only `/api/health_check` and `/api/v1/*`, + or are there additional intended routes that are not documented here? + +## Next Actions + +- Map the deployed behavior against the exact upstream commit shipped in the + running image. +- Confirm whether the deployed image matches the current upstream router and + middleware layout. +- Check whether unauthorized responses should map to `401`, `403`, or `404` + instead of `500`. +- Continue probing only the documented `/api/health_check` and `/api/v1/*` + surfaces unless new evidence suggests extra routes. diff --git a/docs/security/reviews/2026-04/03-http-and-udp-tracker.md b/docs/security/reviews/2026-04/03-http-and-udp-tracker.md new file mode 100644 index 0000000..44fbc85 --- /dev/null +++ b/docs/security/reviews/2026-04/03-http-and-udp-tracker.md @@ -0,0 +1,118 @@ +# HTTP and UDP Tracker - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- HTTP announce and scrape behavior +- UDP announce and scrape behavior +- Parser robustness and malformed input handling +- IP attribution and spoofing risks +- Abuse controls and rate limits + +## Hypotheses + +- The public tracker endpoints may accept malformed requests that reveal parser + or bounds-handling weaknesses. +- Proxy-header trust for HTTP trackers may allow incorrect client IP + attribution if upstream trust is weak. + +## Evidence Reviewed + +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [../../../server/opt/torrust/storage/tracker/etc/tracker.toml](../../../server/opt/torrust/storage/tracker/etc/tracker.toml) +- [../../../docs/infrastructure.md](../../../docs/infrastructure.md) +- Upstream tracker source: + - `packages/axum-http-tracker-server/src/v1/extractors/announce_request.rs` + - `packages/udp-tracker-server/src/handlers/mod.rs` + - `packages/udp-tracker-server/src/handlers/connect.rs` + - `packages/udp-tracker-server/src/handlers/error.rs` + - `packages/udp-tracker-server/src/error.rs` + - `packages/udp-tracker-core/src/services/connect.rs` + +## Checks Performed + +- Confirmed UDP ports `6969` and `6868` are published directly from the tracker + container. +- Confirmed HTTP tracker ports `7070` and `7071` are exposed internally and + published through Caddy virtual hosts. +- Confirmed `on_reverse_proxy = true` is enabled globally in the tracker + configuration. +- Confirmed live HTTP tracker behavior: + - `/` returns `HTTP 404` + - `/announce` returns `HTTP 200` + - `/health_check` returns `HTTP 200` with body `{"status":"Ok"}` + - `/stats`, `/metrics`, and `/robots.txt` return `HTTP 404` + - `/announce` without query params returns a bencoded failure response + explaining that query params are missing + - `http2.torrust-tracker-demo.com` mirrors `http1.torrust-tracker-demo.com` + for `/announce` and `/health_check` + - `HEAD /announce` returns `HTTP 200` + - `POST /announce` returns `HTTP 405` with `Allow: GET,HEAD` + - `OPTIONS /announce` returns `HTTP 405` with `Allow: GET,HEAD` + - Malformed announce query params still return tracker-level bencoded parse + errors rather than generic server failures +- Confirmed upstream HTTP announce extractor behavior: + - Missing or invalid announce query parameters are converted into + tracker-format bencoded error bodies + - Those parser errors are intentionally returned with `HTTP 200`, not a + generic `5xx` response +- Confirmed live UDP tracker behavior with a standard BitTorrent connect probe: + - IPv4 on `udp1.torrust-tracker-demo.com:6969` returns a valid 16-byte + connect response with matching transaction ID + - Short malformed packets can trigger explicit UDP error frames with action + `3`, transaction ID `0`, and parser messages such as `Couldn't parse + action` and `Invalid action` + - Some other malformed but parseable-looking datagrams still timed out during + probing, including invalid-action and wrong-protocol connect-sized payloads + - IPv6 on `udp1.torrust-tracker-demo.com:6969` timed out from this review + environment +- Confirmed upstream UDP connect and error-handling behavior: + - Connect requests are parsed centrally through `Request::parse_bytes(...)` + before dispatch + - Successful connect responses echo the client transaction ID and return a + generated connection cookie derived from the remote socket fingerprint and + current issue time + - Parse failures are routed into the UDP error handler, which builds a + protocol error response even when no transaction ID is available + +## Findings or Non-Findings + +- No confirmed finding yet. The committed configuration establishes the exposed + protocol surfaces but not their parser safety. +- No new finding yet. The public HTTP tracker appears to expose only the + expected tracker route plus a health endpoint. +- The HTTP tracker route behavior looks comparatively well-bounded: unsupported + methods are rejected with `405`, and malformed requests return tracker-level + error content instead of a generic server failure. +- No new finding yet. The second HTTP tracker host appears to mirror the first + rather than exposing a meaningfully different HTTP surface. +- No confirmed security finding yet from the UDP probe. IPv4 behavior looks like + a normal tracker, and at least some malformed UDP inputs are converted into + bounded protocol error frames instead of crashing or exposing a generic + server failure. +- The live UDP surface does not yet appear fully characterized: some malformed + datagrams received structured error replies, while others timed out despite + the current upstream handler being capable of emitting error responses for + parse failures. + +## Open Questions + +- How does the deployed tracker handle malformed HTTP and UDP announce or scrape + requests? +- Are there request-size or rate controls in the tracker implementation? +- Is the public HTTP health endpoint an intentional part of the demo exposure? +- Why do some invalid UDP datagrams receive explicit error frames while others + time out on the live service? +- Is the UDP IPv6 timeout an intentional deployment constraint or an + availability issue on the advertised host? + +## Next Actions + +- Review tracker source for HTTP and UDP request parsing and error handling. +- Validate live endpoint behavior with non-destructive protocol probes. +- Compare additional malformed UDP request classes against the current upstream + parser and error path. +- Avoid broader active probing unless new evidence suggests parser weakness. diff --git a/docs/security/reviews/2026-04/04-grafana.md b/docs/security/reviews/2026-04/04-grafana.md new file mode 100644 index 0000000..f7b9bfa --- /dev/null +++ b/docs/security/reviews/2026-04/04-grafana.md @@ -0,0 +1,93 @@ +# Grafana - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- Public dashboard exposure +- Login and admin surface +- Anonymous access behavior +- Plugin and version risk +- Secret handling relevant to Grafana + +## Hypotheses + +- The public Grafana hostname may expose both public dashboards and the login + surface. +- Grafana configuration may rely on default auth behavior not visible from the + committed compose file alone. + +## Evidence Reviewed + +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [../../../README.md](../../../README.md) +- Live responses from `/`, `/login`, `/public-dashboards/`, `/api/health`, and + `/robots.txt` on the public Grafana host +- Frontend boot-data settings embedded in the unauthenticated `/login` page + +## Checks Performed + +- Confirmed the public hostname `grafana.torrust-tracker-demo.com` proxies to + `grafana:3000`. +- Confirmed Grafana admin user and password are injected through environment + variables. +- Confirmed the README intentionally advertises public dashboards. +- Confirmed live route behavior: + - `/` returns `302` to `/login` + - `/login` returns `200` + - `/public-dashboards/` returns `302` + - `/api/health` returns `200` + - `/robots.txt` returns `200` +- Confirmed `/api/health` body exposes `database: ok`, version `12.3.1`, and + commit `3a1c80ca7ce612f309fdc99338dd3c5e486339be`. +- Confirmed unauthenticated route behavior: + - `/api/live/ws` returns `401` + - `/api/frontend/settings` returns `401` + - `/api/search` returns `401` + - `/signup` and `/user/password/send-reset-email` are routable and return `200` +- Confirmed unauthenticated protected JSON routes return normal Grafana auth + errors rather than internal server errors: + - `/api/frontend/settings` returns JSON `401` with `auth.unauthorized` + - `/api/search` returns JSON `401` with `auth.unauthorized` +- Confirmed frontend boot data on `/login` includes: + - `anonymousEnabled: false` + - `disableLoginForm: false` + - `disableUserSignUp: true` + - `publicDashboardsEnabled: true` +- Confirmed the same boot data also exposes additional operational details: + - `buildInfo.latestVersion: 12.4.2` + - `buildInfo.hasUpdate: true` + - `pluginAdminEnabled: true` + - Preinstalled app/plugin identifiers for logs, traces, metrics, and + pyroscope drilldown apps +- Confirmed the three public dashboard URLs documented in the repository are + reachable without authentication, although one required a longer timeout. + +## Findings or Non-Findings + +- Confirmed finding recorded: the public Grafana host exposes operational + metadata before auth through `/api/health` and login-page boot data. +- No confirmed finding yet. Current evidence suggests that anonymous Grafana UI + browsing is disabled, self-sign-up is disabled, and public dashboards are the + intended unauthenticated surface. +- Non-finding so far: protected JSON API routes return proper `401` responses + rather than leaking the same data through error handling. + +## Open Questions + +- Is anonymous browsing limited strictly to public dashboards? +- Are any plugins installed beyond the base image? +- Should `/api/health` remain publicly reachable on the demo hostname? +- Does the public password reset route create unnecessary account-management + surface for a demo that does not intend public users to log in? +- Should the login-page boot-data disclosure be reduced for the demo, given + that it reveals upgrade status and preinstalled plugin/app metadata? + +## Next Actions + +- Collect live Grafana auth configuration and plugin details. +- Review the public host behavior and login exposure. +- Decide whether `/api/health` and the login bootstrap data should be reduced + or accepted as part of the public demo design. diff --git a/docs/security/reviews/2026-04/05-ssh-and-host.md b/docs/security/reviews/2026-04/05-ssh-and-host.md new file mode 100644 index 0000000..b153a42 --- /dev/null +++ b/docs/security/reviews/2026-04/05-ssh-and-host.md @@ -0,0 +1,60 @@ +# SSH and Host - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- SSH exposure and authentication policy +- Reachable host services +- User and sudo access +- Firewall and host-level hardening +- Patch and package posture + +## Hypotheses + +- SSH is intentionally exposed on both IPv4 and IPv6, but the actual + authentication policy and host-hardening posture cannot be confirmed from the + repository alone. +- Host-level service exposure may differ from the UFW rule files because + Docker-published ports bypass normal UFW filtering. + +## Evidence Reviewed + +- [../../../server/etc/ufw/user.rules](../../../server/etc/ufw/user.rules) +- [../../../server/etc/ufw/user6.rules](../../../server/etc/ufw/user6.rules) +- [../../../server/etc/ufw/before6.rules](../../../server/etc/ufw/before6.rules) +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../docs/infrastructure.md](../../../docs/infrastructure.md) + +## Checks Performed + +- Confirmed UFW explicitly allows inbound TCP `22` on both IPv4 and IPv6. +- Confirmed UFW explicitly allows inbound UDP `6969` on both IPv4 and IPv6. +- Confirmed the IPv6 rules include a manual SNAT rule for Docker UDP tracker + replies on port `6969`. +- Confirmed the compose comments document that Docker-published ports bypass + the usual UFW filtering model. +- Confirmed no repo-side evidence yet for `sshd_config`, SSH key policy, + password-auth settings, host package updates, or other host service bindings. + +## Findings or Non-Findings + +- No confirmed finding yet from repository evidence alone. The repo confirms + intended SSH exposure and special IPv6/Docker handling, but not the host's + actual authentication, patch, or service-hardening posture. + +## Open Questions + +- Is password authentication disabled in SSH? +- Are root login and sudo access restricted as expected? +- Which host services are actually listening, beyond the repo's intended + configuration? +- Is the host patched and current on security updates? + +## Next Actions + +- Collect `sshd_config`, `ss -tulpn`, and package update status from the live + host. +- Compare the intended UFW rules with actual listening sockets and reachable + services. diff --git a/docs/security/reviews/2026-04/06-container-and-persistence.md b/docs/security/reviews/2026-04/06-container-and-persistence.md new file mode 100644 index 0000000..05f66b4 --- /dev/null +++ b/docs/security/reviews/2026-04/06-container-and-persistence.md @@ -0,0 +1,63 @@ +# Container and Persistence - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- Container privilege model +- Mounted volumes and writable paths +- Secret exposure through runtime configuration +- Lateral movement between services +- Persistence via cron, backups, or writable config + +## Hypotheses + +- A compromise of the tracker or Grafana container could become persistent + through writable mounted storage. +- Network segmentation is partial rather than strict, and compromise of one + internet-facing service may provide access to internal services. + +## Evidence Reviewed + +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../server/opt/torrust/storage/backup/etc/backup-paths.txt](../../../server/opt/torrust/storage/backup/etc/backup-paths.txt) +- [../../../server/etc/cron.d/tracker-backup](../../../server/etc/cron.d/tracker-backup) +- [../../../server/usr/local/bin/maintenance-backup.sh](../../../server/usr/local/bin/maintenance-backup.sh) + +## Checks Performed + +- Confirmed the tracker has writable mounts for config, logs, and data. +- Confirmed Grafana and MySQL use persistent storage. +- Confirmed the backup workflow reads configuration and database state and + writes archives under persistent storage. +- Confirmed tracker spans `metrics_network`, `database_network`, and + `proxy_network`, making it the most connected internet-facing service. + +## Findings or Non-Findings + +- No confirmed finding yet. The current configuration creates several + persistence and lateral-movement paths that need runtime and source review. +- Non-finding from static config review: the repository shows some deliberate + network separation between proxy, database, metrics, and visualization + networks, even though the tracker remains the most connected internet-facing + service. +- Non-finding from static config review: the backup workflow is explicit and + auditable rather than hidden, but it still needs runtime evidence to confirm + file ownership, container users, and writable-path constraints. + +## Open Questions + +- Which services run as root inside their containers? +- Are any mounts writable beyond what the service strictly needs? +- Can a tracker compromise reach MySQL or Prometheus with useful credentials? +- Which files under the mounted storage paths are writable by each service at + runtime? + +## Next Actions + +- Review container users, capabilities, and network reachability from runtime + evidence. +- Review backup scripts for secret exposure and privilege-escalation paths. +- Collect `docker ps`, `docker inspect`, and runtime mount information from the + live host to verify the actual privilege model. diff --git a/docs/security/reviews/2026-04/07-supply-chain.md b/docs/security/reviews/2026-04/07-supply-chain.md new file mode 100644 index 0000000..60d9a00 --- /dev/null +++ b/docs/security/reviews/2026-04/07-supply-chain.md @@ -0,0 +1,68 @@ +# Supply Chain - 2026-04 + +**Summary**: [README.md](README.md) +**Progress**: [progress.md](progress.md) + +## Scope + +- Image provenance and digests +- Mutable tags and version pinning +- Upstream revisions and release mapping +- Known CVEs affecting deployed versions +- Build and deployment traceability + +## Hypotheses + +- Mutable image tags reduce deployment reproducibility and complicate incident + response. +- The exact source revision backing the deployed tracker image may be unknown. + +## Evidence Reviewed + +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../docs/infrastructure.md](../../../docs/infrastructure.md) +- Upstream issue: `torrust/torrust-tracker-deployer#254` + +## Checks Performed + +- Confirmed the tracker image is `torrust/tracker:develop`. +- Confirmed the backup image is `torrust/tracker-backup:latest`. +- Confirmed other core services are version-pinned by tag in compose: Caddy, + Prometheus, Grafana, and MySQL. +- Confirmed upstream deployer issue `#254` explicitly plans to replace + `torrust/tracker:develop` with `torrust/tracker:v4.0.0` after the stable + tracker release. +- Confirmed the upstream rationale for that change matches this review: the + `develop` tag is considered unsuitable for production because it can change + unexpectedly and should only be updated after manual compatibility + verification. +- Confirmed a quick upstream issue search did not reveal a backup-image pinning + issue equivalent to tracker issue `#254`; the closest related results were + the broader Docker image refresh issue `#317` and backup-image CVE follow-up + issue `#431`. + +## Findings or Non-Findings + +- Confirmed finding recorded: the deployed compose config uses mutable image + tags for the tracker and backup services, which reduces deployment + reproducibility and weakens rollback and incident-response traceability. +- The tracker-tag portion of this finding is not speculative; the deployer + project already tracks it as a blocked follow-up awaiting a stable tracker + release. + +## Open Questions + +- Which image digests are currently deployed? +- Which tracker source revision corresponds to the deployed `develop` image? +- Are there known CVEs affecting the exact deployed image digests? +- Should a dedicated upstream task be opened to pin + `torrust/tracker-backup:latest` with the same level of traceability as the + tracker image? + +## Next Actions + +- Collect live image digests and deployed revisions. +- Review upstream release and image provenance for the deployed services. +- Verify which digests are currently cached or running on the live host. +- Track whether `torrust/torrust-tracker-deployer#254` is implemented and + whether a corresponding backup-image pinning task should be added upstream. diff --git a/docs/security/reviews/2026-04/README.md b/docs/security/reviews/2026-04/README.md new file mode 100644 index 0000000..1bc5b52 --- /dev/null +++ b/docs/security/reviews/2026-04/README.md @@ -0,0 +1,79 @@ +# Security Review Summary - 2026-04 + +**Plan**: [../../security-review-plan.md](../../security-review-plan.md) +**Issue**: [#13](https://github.com/torrust/torrust-tracker-demo/issues/13) +**Status**: In progress. Static-config and public-surface review completed; +host-runtime evidence still pending. + +## Review Metadata + +- Review cycle: `2026-04` +- Reviewer: GitHub Copilot with repository maintainer input +- Start date: 2026-04-09 +- End date: + +## Scope Covered + +- [x] Caddy and HTTPS routing +- [x] Tracker API +- [x] HTTP and UDP tracker protocol surface +- [x] Grafana +- [ ] SSH and host exposure +- [x] Container, persistence, and lateral movement +- [x] Supply chain and deployment provenance + +## Deployment Summary + +- Repository revision: `7e57cddf091fc8e388b0acb891a66154f456a296` +- Deployed tracker revision: +- Container image digests reviewed: +- Host runtime summary: Repository configuration reviewed only. Live runtime + evidence not collected yet. + +## Evidence Reviewed + +- Repository configuration in `server/` +- [README.md](../../../README.md) +- [../../security-review-plan.md](../../security-review-plan.md) +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [../../../server/opt/torrust/storage/tracker/etc/tracker.toml](../../../server/opt/torrust/storage/tracker/etc/tracker.toml) +- [../../../docs/infrastructure.md](../../../docs/infrastructure.md) +- [../../../server/etc/ufw/user.rules](../../../server/etc/ufw/user.rules) +- [../../../server/etc/ufw/user6.rules](../../../server/etc/ufw/user6.rules) +- [../../../server/etc/ufw/before6.rules](../../../server/etc/ufw/before6.rules) + +## Confirmed Findings + +- The public Grafana host exposes operational metadata before auth through + `/api/health` and login boot data. +- The public HTTPS hosts redirect HTTP to HTTPS but do not advertise HSTS, + leaving a low-severity edge-hardening gap for first-visit clients. +- The deployed compose config uses mutable image tags for the tracker and + backup services, which reduces deployment traceability and rollback + confidence. +- Public tracker API host requests return `500` and expose internal auth error + text instead of proper client errors, including on apparently unrelated paths. + +## Rejected Hypotheses + +- None recorded yet. + +## Follow-Up Actions + +- Request live runtime evidence listed in `progress.md`. +- Obtain the exact deployed tracker revision. +- Collect the exact image digests currently deployed for the tracker and backup + services. +- Continue source-backed review of the public API, edge behavior, and tracker + request handling. +- Confirm whether the deployed tracker image matches the current upstream API + router and auth middleware layout. + +## Open Questions + +- Which exact tracker source revision backs the deployed `torrust/tracker:develop` + image? +- Which exact image digests are currently running for the tracker and backup + services? +- What SSH authentication policy is active on the host? diff --git a/docs/security/reviews/2026-04/findings.md b/docs/security/reviews/2026-04/findings.md new file mode 100644 index 0000000..8971f89 --- /dev/null +++ b/docs/security/reviews/2026-04/findings.md @@ -0,0 +1,173 @@ +# Security Review Findings - 2026-04 + +**Summary**: [README.md](README.md) + +Use this file only for confirmed findings and explicitly accepted risks. +Hypotheses, dead ends, and exploratory notes belong in the per-surface review +files. + +## Confirmed Findings + +### Finding: Public Grafana host discloses operational metadata before auth + +- Severity: Low +- Surface: Grafana public exposure +- Preconditions: Unauthenticated access to the public Grafana hostname +- Attack path: Request public routes such as `/api/health` and `/login` on + `https://grafana.torrust-tracker-demo.com/` and inspect the returned JSON and + frontend boot data +- Evidence: + - `GET https://grafana.torrust-tracker-demo.com/api/health` returns + `HTTP/2 200` and exposes `database: ok`, version `12.3.1`, and commit + `3a1c80ca7ce612f309fdc99338dd3c5e486339be` + - The unauthenticated `/login` page exposes Grafana boot-data flags showing + `anonymousEnabled: false`, `disableLoginForm: false`, + `disableUserSignUp: true`, `publicDashboardsEnabled: true`, + `pluginAdminEnabled: true`, `latestVersion: 12.4.2`, and `hasUpdate: true` + - The same boot data exposes edition and plugin/app metadata, including the + preinstalled drilldown apps + - Protected JSON routes such as `/api/frontend/settings` and `/api/search` + return proper `401` responses, which shows the disclosure is coming from + intentionally public routes rather than from a server error path +- Impact: The public host gives unauthenticated visitors a clearer fingerprint + of the Grafana deployment, including exact version, commit, update lag, and + enabled feature or plugin surface, which can help attackers prioritize known + issues or tailor follow-on probing. +- Remediation: Restrict or proxy-filter public Grafana routes that are not + needed for the demo, especially `/api/health`, and reduce unauthenticated + boot-data exposure where practical. +- Status: Open + +### Finding: Public HTTPS hosts do not advertise HSTS + +- Severity: Low +- Surface: Edge HTTPS hardening +- Preconditions: A user's first or non-pinned visit to one of the public HTTPS + hosts over an attacker-controlled or hostile network +- Attack path: Induce the client to connect over plaintext HTTP first or strip + the upgrade before HSTS has ever been cached; because the public hosts do not + advertise `Strict-Transport-Security`, the browser is not instructed to pin + HTTPS for future visits +- Evidence: + - [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) + explicitly publishes port `80` as `# HTTP (ACME HTTP-01 challenge)` for the + Caddy service + - `http://api.torrust-tracker-demo.com/`, + `http://http1.torrust-tracker-demo.com/`, + `http://http2.torrust-tracker-demo.com/`, and + `http://grafana.torrust-tracker-demo.com/` all redirect to HTTPS with + `HTTP 308` + - Tested HTTPS root responses for `api.torrust-tracker-demo.com`, + `http1.torrust-tracker-demo.com`, `http2.torrust-tracker-demo.com`, and + `grafana.torrust-tracker-demo.com` did not include a + `Strict-Transport-Security` header + - [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) + contains reverse-proxy site blocks for the public hosts and automatic + Let's Encrypt handling, but no header directive that would add HSTS +- Impact: Plaintext-to-HTTPS redirect behavior is present, but first-visit + users remain more exposed to downgrade or SSL-stripping attacks than they + would be with HSTS enabled. +- Remediation: Add an HSTS policy on the public HTTPS hosts, ideally with a + conservative initial max-age that can be increased after verification. This + does not require closing port `80`; the current config explicitly keeps port + `80` open for ACME HTTP-01 certificate issuance and renewal. +- Status: Open + +### Finding: Mutable container tags reduce deployment traceability + +- Severity: Low +- Surface: Supply chain and deployment provenance +- Preconditions: Access to the deployment process or routine image refresh, + redeploy, or host rebuild +- Attack path: Reuse the committed compose configuration as-is and pull + `torrust/tracker:develop` or `torrust/tracker-backup:latest` at a later time; + the host can receive different image contents without any configuration-file + change, leaving operators unable to identify the exact deployed code from the + tracked server config alone +- Evidence: + - [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) + sets the tracker image to `torrust/tracker:develop` + - The same compose file sets the backup image to + `torrust/tracker-backup:latest` + - The tracker service block includes an inline comment stating that the + `develop` tag is mutable and introduces deployment non-reproducibility + - Upstream deployer issue `torrust/torrust-tracker-deployer#254` explicitly + tracks replacing `torrust/tracker:develop` with `torrust/tracker:v4.0.0` + after the stable tracker release, and documents that `develop` is not a + good choice for production deployments because it can introduce breaking + changes at any time + - Other core services in the same compose file are pinned to explicit version + tags, which shows the mutable tags are an exception rather than the general + deployment pattern +- Impact: Mutable tags weaken release traceability, make rollback and incident + response harder, and increase the chance of unintentionally deploying code + that differs from what operators believe is running. +- Remediation: Pin deployed images to immutable digests or at least fixed + release tags, and record the exact deployed revision in the review evidence. + The already-open upstream path is to switch the tracker image from + `develop` to `v4.0.0` once that stable release is available. +- Status: Open. Upstream already tracks the tracker-tag remediation in + `torrust/torrust-tracker-deployer#254`, but the live demo config still uses a + mutable tracker tag today and the backup image remains on `latest`. + +### Finding: Public tracker API host returns 500 with internal auth error text + +- Severity: Low +- Surface: Tracker API +- Preconditions: Unauthenticated access to the public API hostname +- Attack path: Send an unauthenticated request to the public API hostname on a + protected route or even an apparently unrelated path such as + `https://api.torrust-tracker-demo.com/api/v1/stats` or + `https://api.torrust-tracker-demo.com/swagger` +- Evidence: + - `http://api.torrust-tracker-demo.com/` redirects to + `https://api.torrust-tracker-demo.com/` with `HTTP/1.1 308 Permanent Redirect` + - `GET /api/v1/stats`, `/api/v1/torrents`, `/api/v1/whitelist`, and + `/api/v1/keys` all return `HTTP/2 500` + - Exact `GET /api/health_check` returns `HTTP/2 200`, but + `GET /api/health_check/` and `GET /api/health_check/foo` return the same + auth-shaped `HTTP/2 500` + - `GET /api/v1/notfound`, `/api/v1/foo/bar`, `/api/notfound`, `/swagger`, + and `/` also return `HTTP/2 500` + - Response bodies expose distinct internal auth failure strings: + `unauthorized`, `token not valid`, and `unknown token provided` + - The HTTPS API responses are served through the public Caddy edge and + include public-facing headers such as `via: 1.1 Caddy` and `x-request-id` + - Upstream router composition adds the versioned API routes first, wraps that + router with auth middleware, and only then adds the exact public + `/api/health_check` route + - Upstream auth middleware returns `Unauthorized`, `TokenNotValid`, and + `UnknownTokenProvided` through `unhandled_rejection_response(...)` + - The same middleware returns early on missing or invalid auth data instead + of always calling `next.run(request).await` + - Upstream router exposes `GET /api/health_check` separately, and the live + deployment returns `HTTP/2 200` for that path +- Invalid bearer tokens also change unmatched-path responses from + `unauthorized` to `token not valid`, which is consistent with the current + auth-wrapped router intercepting requests before a downstream `404` +- Impact: The service exposes internal error text and uses a server error for + authorization failures, which leaks implementation behavior and complicates + monitoring, alerting, incident triage, and client-side handling of auth + errors. +- Remediation: Map unauthorized requests to the appropriate client error + response and avoid returning internal exception text in the response body. +- Status: Open. Public-edge behavior is confirmed, and current upstream router + composition likely explains the broad auth-shaped failures; the remaining + runtime question is whether the deployed image matches that upstream layout. + +## Accepted Risks + +No accepted risks recorded yet. + +## Finding Template + +### Finding: <title> + +- Severity: +- Surface: +- Preconditions: +- Attack path: +- Evidence: +- Impact: +- Remediation: +- Status: diff --git a/docs/security/reviews/2026-04/progress.md b/docs/security/reviews/2026-04/progress.md new file mode 100644 index 0000000..30c9043 --- /dev/null +++ b/docs/security/reviews/2026-04/progress.md @@ -0,0 +1,203 @@ +# Security Review Progress - 2026-04 + +**Plan**: [../../security-review-plan.md](../../security-review-plan.md) +**Summary**: [README.md](README.md) +**Findings**: [findings.md](findings.md) + +## Review Status + +| Surface | Status | Notes | +| ------- | ------ | ----- | +| Caddy and HTTPS | In progress | Confirmed low-severity finding: public hosts redirect HTTP to HTTPS but do not advertise HSTS | +| Tracker API | In progress | Public exact `/api/health_check` returns 200; most other tested API-host paths, including path variants and unmatched paths, return auth-shaped 500 responses | +| HTTP and UDP tracker | In progress | Both HTTP tracker hosts mirror expected announce and health behavior; UDP IPv4 responds to connect probes and some malformed packets get bounded error frames; IPv6 timed out | +| Grafana | In progress | Confirmed low-severity finding: public host exposes `/api/health` and login boot-data metadata; login form is enabled while anonymous browsing is disabled | +| SSH and host | Blocked | Repository evidence reviewed; host authentication and service posture need runtime data | +| Container and persistence | In progress | Static config review complete; runtime privilege and mount details still needed | +| Supply chain | In progress | Confirmed low-severity finding: tracker and backup images use mutable tags in deployed compose config | + +## Evidence Requested + +- [ ] Tracker source repository and deployed revision +- [ ] Redacted `.env` +- [ ] `docker ps` +- [ ] `docker inspect` output +- [ ] `docker network inspect` output +- [ ] `ss -tulpn` +- [ ] `ufw status verbose` +- [ ] `sshd_config` +- [ ] Grafana auth and plugin details +- [ ] OS package update status + +## Evidence Received + +- Repository configuration under `server/` +- [../../../README.md](../../../README.md) +- [../../../docs/infrastructure.md](../../../docs/infrastructure.md) +- [../../../server/opt/torrust/docker-compose.yml](../../../server/opt/torrust/docker-compose.yml) +- [../../../server/opt/torrust/storage/caddy/etc/Caddyfile](../../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [../../../server/opt/torrust/storage/tracker/etc/tracker.toml](../../../server/opt/torrust/storage/tracker/etc/tracker.toml) +- [../../../server/etc/ufw/user.rules](../../../server/etc/ufw/user.rules) +- [../../../server/etc/ufw/user6.rules](../../../server/etc/ufw/user6.rules) +- [../../../server/etc/ufw/before6.rules](../../../server/etc/ufw/before6.rules) +- Live HTTP response headers and bodies for `api.torrust-tracker-demo.com`, + `grafana.torrust-tracker-demo.com`, and `http1.torrust-tracker-demo.com` +- Focused path enumeration results for the API host, Grafana host, and HTTP + tracker host +- Upstream tracker source snippets for the API router, auth middleware, and + health-check handler +- Upstream tracker source snippets for HTTP announce extraction and UDP packet, + connect, and error handling + +## Working Notes + +- Public surfaces visible from repo config: Caddy on `80/443`, SSH on `22`, + UDP tracker on `6969` and `6868`, public Grafana through Caddy, and tracker + API through Caddy. +- The tracker container uses the mutable image tag `torrust/tracker:develop`. +- The backup service uses `torrust/tracker-backup:latest`, which is also + mutable. +- The supply-chain review now has a confirmed low-severity finding because the + deployed compose config uses mutable tags for the tracker and backup + services, weakening deployment traceability. +- Upstream deployer issue `torrust/torrust-tracker-deployer#254` already tracks + replacing `torrust/tracker:develop` with `torrust/tracker:v4.0.0` after the + stable release, which strengthens the rationale for the tracker-tag portion + of the finding. +- A quick upstream issue search did not reveal an equivalent dedicated task for + pinning `torrust/tracker-backup:latest`; the nearest hits were the broader + image-update issue `#317` and backup-image CVE issue `#431`. +- Tracker, Caddy, Grafana, and backup all rely on mounted persistent storage. +- HTTP tracker routes trust reverse-proxy headers for client IP attribution. +- The latest edge review pass promoted missing HSTS on the public HTTPS hosts + from an open hardening question to a confirmed low-severity finding. +- Live checks observed: + - `https://api.torrust-tracker-demo.com/` returns `HTTP/2 500` + - `https://api.torrust-tracker-demo.com/health_check` returns `HTTP/2 500` + - `https://api.torrust-tracker-demo.com/api/health_check` returns `HTTP/2 200` + - `https://api.torrust-tracker-demo.com/api/health_check/` returns `HTTP/2 500` + - `https://api.torrust-tracker-demo.com/api/health_check/foo` returns `HTTP/2 500` + - `http://api.torrust-tracker-demo.com/` returns `HTTP/1.1 308 Permanent Redirect` + - API root body exposes `Unhandled rejection: Err { reason: "unauthorized" }` + - Tested API paths `/login`, `/api`, `/api/`, `/stats`, `/metrics`, + `/health_check`, `/announce`, `/swagger`, `/openapi.json`, and + `/robots.txt` all return `HTTP 500` + - Tested protected API paths `/api/v1/stats`, `/api/v1/torrents`, + `/api/v1/whitelist`, and `/api/v1/keys` all return `HTTP 500` without a + token + - Clearly unmatched paths `/api/v1/notfound`, `/api/v1/foo/bar`, + `/api/notfound`, `/swagger`, and `/` also return `HTTP 500` + - `/api/v1/stats` also returns distinct internal error strings for different + auth failures: `unauthorized`, `token not valid`, and + `unknown token provided` + - Invalid bearer tokens also change the response bodies on unmatched paths + from `unauthorized` to `token not valid` + - `https://grafana.torrust-tracker-demo.com/` returns `HTTP/2 302` with + `Location: /login` + - `http://grafana.torrust-tracker-demo.com/` returns `HTTP/1.1 308 Permanent Redirect` + - `https://grafana.torrust-tracker-demo.com/login` returns `HTTP/2 200` + - `https://grafana.torrust-tracker-demo.com/api/health` returns `HTTP/2 200` + - Grafana `/api/health` body exposes database status, version `12.3.1`, and + commit `3a1c80ca7ce612f309fdc99338dd3c5e486339be` + - Grafana root responses include `x-content-type-options: nosniff` and + `x-frame-options: deny` + - Grafana frontend boot data on `/login` exposes: + - `anonymousEnabled: false` + - `disableLoginForm: false` + - `disableUserSignUp: true` + - `publicDashboardsEnabled: true` + - `buildInfo.latestVersion: 12.4.2` + - `buildInfo.hasUpdate: true` + - `pluginAdminEnabled: true` + - The Grafana review now has a confirmed low-severity finding because those + public routes disclose version, commit, update, and feature metadata before + authentication + - Two documented public dashboard URLs returned `HTTP 200` + - The third documented public dashboard URL loaded after a longer timeout + - Grafana unauthenticated routes `/api/live/ws`, `/api/frontend/settings`, + and `/api/search` returned `401` + - Grafana protected JSON routes return standard JSON `401` responses with + `auth.unauthorized`, not internal server errors + - `https://http1.torrust-tracker-demo.com/` returns `HTTP/2 404` + - `http://http1.torrust-tracker-demo.com/` returns `HTTP/1.1 308 Permanent Redirect` + - `https://http1.torrust-tracker-demo.com/announce` returns `HTTP/2 200` + - `https://http1.torrust-tracker-demo.com/health_check` returns `HTTP/2 200` + with body `{"status":"Ok"}` + - `https://http2.torrust-tracker-demo.com/announce` returns `HTTP/2 200` + - `https://http2.torrust-tracker-demo.com/health_check` returns `HTTP/2 200` + with body `{"status":"Ok"}` + - `https://http1.torrust-tracker-demo.com/announce` without query params + returns a bencoded failure response describing missing query params + - `HEAD https://http1.torrust-tracker-demo.com/announce` returns `HTTP/2 200` + - `OPTIONS https://http1.torrust-tracker-demo.com/announce` returns `HTTP/2 405` + with `Allow: GET,HEAD` + - `POST https://http1.torrust-tracker-demo.com/announce` returns `HTTP/2 405` + with `Allow: GET,HEAD` + - Malformed announce query params return bencoded parser errors, including + invalid `info_hash` length diagnostics + - Upstream HTTP announce extraction intentionally returns those parser errors + as tracker-formatted bencoded bodies on `HTTP 200` + - A valid BitTorrent UDP connect probe to IPv4 `udp1.torrust-tracker-demo.com:6969` + returns a 16-byte connect response with the expected transaction ID + - Additional UDP probes showed more specific malformed-input behavior: + - An 8-byte garbage packet returned a UDP error response with action `3`, + transaction ID `0`, and parser text `Couldn't parse action` + - A 12-byte garbage packet returned a UDP error response with action `3`, + transaction ID `0`, and parser text `Invalid action` + - Invalid-action, wrong-protocol, and random 16-byte payloads timed out in + this review environment + - The same valid UDP connect probe to the advertised IPv6 address timed out + - Upstream UDP `handle_packet(...)` parses requests through + `Request::parse_bytes(...)` and routes parse failures into the UDP error + handler + - Upstream UDP connect handling echoes the request transaction ID and derives + the returned connection cookie from the remote socket fingerprint and issue + time + - No `Strict-Transport-Security` header was observed on the tested root + responses for the API, HTTP tracker, or Grafana hosts + - `OPTIONS https://api.torrust-tracker-demo.com/` still returns `HTTP/2 500` +- Upstream API source observations: + - `packages/axum-rest-tracker-api-server/src/routes.rs` exposes public + `GET /api/health_check` + - The same router adds the versioned API routes first, wraps that router with + auth middleware, and only then adds the exact public + `GET /api/health_check` route + - `packages/axum-rest-tracker-api-server/src/v1/middlewares/auth.rs` maps + missing or invalid tokens to `unhandled_rejection_response(...)` + - The auth middleware returns early on missing or invalid auth data instead + of always calling `next.run(request).await` + - The current upstream behavior therefore explains the live `500` + unauthorized response on protected v1 paths and on API-host paths that fall + into the auth-wrapped router instead of the exact public health route + - The committed Caddy config does not rewrite the API-host request path + - The remaining runtime question is narrower: whether the deployed image + matches the current upstream router and middleware layout + +## Blockers + +- Live runtime evidence has not been collected yet. +- The deployed tracker source revision is still unknown. +- The remaining host, SSH, image-digest, and container-privilege questions + cannot be resolved further from repository and public-endpoint evidence alone. + +## Next Actions + +- Request live runtime evidence from the operator. +- Obtain the exact deployed tracker revision. +- Collect the exact image digests currently deployed for the tracker and backup + services. +- Continue source-backed review where runtime behavior still diverges from the + current upstream code. +- Confirm whether the deployed tracker image matches the current upstream API + router and auth middleware layout. +- Decide whether a dedicated upstream task should be opened to pin the backup + image away from `torrust/tracker-backup:latest`. +- Decide whether HSTS should be added directly in Caddy or through the + deployer-generated template path. +- Decide whether `/api/health` and login boot-data disclosure should be reduced + or explicitly accepted as part of the public demo design. +- Determine whether the live UDP timeout cases reflect expected parser or + request-class handling, packet loss, or deployed-runtime divergence from + current upstream behavior. +- Determine whether the UDP IPv6 timeout is an intentional limitation or an + operational regression on the public tracker host. diff --git a/docs/security/reviews/README.md b/docs/security/reviews/README.md new file mode 100644 index 0000000..6dbafba --- /dev/null +++ b/docs/security/reviews/README.md @@ -0,0 +1,22 @@ +# Security Review Reports + +This folder contains recurring security review reports for the live Torrust +Tracker Demo deployment. + +## Conventions + +- Use one folder per review cycle. +- Prefer time-based names such as `2026-04` or `2026-q2`. +- Keep sensitive raw evidence out of git. +- Commit sanitized summaries, findings, and references to private evidence + instead. + +## Standard Review-Cycle Files + +- `README.md` for the final summary report. +- `progress.md` for in-progress tracking. +- `findings.md` for confirmed findings and accepted risks. +- One numbered file per attack surface or service under review. + +The reporting model is defined in +[../security-review-plan.md](../security-review-plan.md). diff --git a/docs/security/security-review-plan.md b/docs/security/security-review-plan.md new file mode 100644 index 0000000..0c9ed8e --- /dev/null +++ b/docs/security/security-review-plan.md @@ -0,0 +1,414 @@ +# Security Review Plan + +This document defines a repeatable security review process for the live Torrust +Tracker Demo deployment. The goal of the review is to identify realistic paths +an attacker could use to gain initial access to the demo server, a running +container, or privileged application functionality, and to document the +evidence, impact, and remediation for each finding. + +This is a living document. It should be re-evaluated periodically and updated +after meaningful infrastructure or application changes. + +## Review Goal + +The primary question for this review is: + +> How could an external attacker obtain meaningful access to the demo server or +> its deployed services? + +For this plan, meaningful access includes any of the following outcomes: + +- Host shell access. +- Access to a running container with useful persistence or secrets. +- Access to privileged application functionality such as admin APIs or Grafana + administration. +- Access to sensitive data such as credentials, tokens, database contents, or + configuration not intended to be public. +- The ability to modify service behavior, deployed configuration, or persistent + state. + +## Scope + +The review covers the full public-facing tracker demo deployment described in +[README.md](../../README.md), [infrastructure.md](../infrastructure.md), and +the configuration under [server/](../../server/README.md). + +### In scope + +- Public HTTP and HTTPS endpoints served through Caddy. +- Public UDP tracker endpoints. +- Public Grafana exposure and dashboard-sharing configuration. +- Public tracker API exposure and authentication model. +- SSH access and host-level hardening. +- Docker Compose topology, container isolation, volumes, and network exposure. +- Secret handling through environment variables, mounted files, and backups. +- Supply-chain risks in container images and deployment artifacts. +- Misconfiguration that enables lateral movement from one service to another. + +### Out of scope by default + +- Denial-of-service resistance and capacity testing. +- Social engineering. +- Third-party provider compromise such as Hetzner control-plane compromise. +- Vulnerabilities in services not deployed on this demo server. + +These items can be added explicitly when needed, but they are not required for +the baseline recurring review. + +## Review Cadence + +Run this review at the following times: + +- Quarterly. +- After major infrastructure changes. +- After public exposure of a new endpoint, hostname, or port. +- After changes to authentication, secrets, Docker networking, or firewall + rules. +- After upgrading core services such as the tracker, Caddy, Grafana, Docker, + MySQL, or the OS. +- After any security incident, suspicious activity, or near miss. + +## Current Deployment Summary + +At the time this plan was written, the deployment exposes the following public +surfaces: + +- HTTPS via Caddy on ports `80` and `443`. +- UDP tracker on port `6969`. +- UDP tracker on port `6868`. +- SSH on port `22`. +- Public Grafana through Caddy. +- Public tracker API through Caddy. + +Relevant configuration references: + +- [server/opt/torrust/docker-compose.yml](../../server/opt/torrust/docker-compose.yml) +- [server/opt/torrust/storage/caddy/etc/Caddyfile](../../server/opt/torrust/storage/caddy/etc/Caddyfile) +- [server/opt/torrust/storage/tracker/etc/tracker.toml](../../server/opt/torrust/storage/tracker/etc/tracker.toml) +- [server/etc/ufw/user.rules](../../server/etc/ufw/user.rules) +- [server/etc/ufw/user6.rules](../../server/etc/ufw/user6.rules) +- [server/etc/ufw/before6.rules](../../server/etc/ufw/before6.rules) +- [infrastructure.md](../infrastructure.md) + +## Review Method + +The review should proceed in phases so that configuration review, source review, +and runtime validation reinforce each other. + +## Review Workspace Layout + +Each review cycle should have its own folder under `docs/security/reviews/` so +that in-progress work and final reporting stay grouped together. + +Recommended structure: + +```text +docs/security/reviews/ +└── <review-cycle>/ + ├── README.md + ├── progress.md + ├── findings.md + ├── 01-caddy-and-https.md + ├── 02-tracker-api.md + ├── 03-http-and-udp-tracker.md + ├── 04-grafana.md + ├── 05-ssh-and-host.md + ├── 06-container-and-persistence.md + └── 07-supply-chain.md +``` + +Naming convention: + +- Use one folder per review cycle. +- Prefer a time-based folder name such as `2026-04` or `2026-q2`. +- Keep raw sensitive evidence out of git. Only commit sanitized excerpts, + conclusions, and references to privately stored evidence when needed. + +Purpose of each file: + +- `README.md` is the final summary report for the review cycle. +- `progress.md` tracks work in progress, evidence requested, status by attack + surface, and open questions. +- `findings.md` is the consolidated list of confirmed findings and accepted + risks. +- The numbered surface files hold the detailed review notes for each exposed + service or attack surface. + +## Phase 1: Baseline Inventory + +Build an inventory of the live environment before testing assumptions. + +Checklist: + +- Confirm all public DNS records and host names. +- Confirm all listening ports on the host. +- Confirm all Docker-published ports. +- Confirm which services are intentionally public and which are internal only. +- Record exact container image tags and image digests. +- Record exact deployed application revisions where available. +- Record OS version, Docker version, and package update status. + +Primary evidence to collect: + +- `docker ps` +- `docker inspect <container>` +- `docker network inspect <network>` +- `ss -tulpn` +- `ufw status verbose` +- `iptables-save` +- `ip6tables-save` +- Exact deployment `.env` variable names with secret values redacted + +## Phase 2: External Attack Surface Review + +Review every network-reachable entry point as if no credentials are available. + +### Caddy and HTTPS routes + +Review goals: + +- Confirm only intended virtual hosts are exposed. +- Confirm no administrative interface is reachable. +- Confirm header trust boundaries are correct. +- Confirm reverse proxy configuration does not unintentionally expose backend + endpoints. + +Checks: + +- Enumerate configured host names and path routing. +- Review how `X-Forwarded-For` and related headers are trusted by the tracker. +- Check whether unexpected host headers or fallback routes reach a backend. +- Check TLS configuration, redirects, and certificate handling. + +### Tracker API + +Review goals: + +- Identify all public endpoints. +- Confirm which endpoints require authentication. +- Confirm whether admin token use is minimal and correctly scoped. + +Checks: + +- Review all routes and methods in the tracker source code. +- Review token parsing, storage, comparison, and logging behavior. +- Identify any debug, diagnostics, metrics, or file-serving endpoints. +- Verify whether malformed requests can trigger panics, unexpected disclosure, + or state changes. + +### Grafana + +Review goals: + +- Confirm that public access is limited to the intended dashboard-sharing model. +- Confirm admin access is not exposed accidentally. +- Confirm secrets and plugins do not expand the attack surface unnecessarily. + +Checks: + +- Review Grafana authentication and anonymous-access configuration. +- Review public dashboard exposure. +- Confirm whether login is enabled on the public hostname. +- Review installed plugins and their provenance. +- Review known security advisories for the deployed Grafana version. + +### HTTP and UDP tracker endpoints + +Review goals: + +- Identify parser, protocol, and request-handling risks. +- Confirm that tracker endpoints do not expose administrative behavior. + +Checks: + +- Review announce and scrape handling in the tracker source. +- Review request parsing, bounds handling, and error paths. +- Review whether peer IP handling can be spoofed through proxy headers. +- Review rate limiting and abuse controls. +- Review whether malformed UDP or HTTP requests can crash the service or lead + to unexpected state. + +### SSH + +Review goals: + +- Confirm the host does not present a low-effort initial access path. + +Checks: + +- Review `sshd_config`. +- Confirm whether password authentication is disabled. +- Confirm whether root login is disabled. +- Confirm which users have shell access and sudo rights. +- Confirm whether rate limiting, logging, and alerting are in place. + +## Phase 3: Source Code Review + +Configuration review alone is not enough. Review the source code for every +publicly reachable custom service or custom wrapper. + +Priority order: + +1. Tracker source code. +2. Any custom deployment scripts or wrappers that handle secrets, backups, or + service startup. +3. Any custom Grafana provisioning or Caddy templating logic if the deployed + runtime differs from this repository. + +Source review goals: + +- Identify authentication and authorization boundaries. +- Identify unsafe parsing and deserialization paths. +- Identify sensitive endpoints not obvious from configuration. +- Identify filesystem, shell, subprocess, SSRF, SQL, or template injection + risks. +- Identify unsafe assumptions about proxy headers and client identity. + +## Phase 4: Container and Host Hardening Review + +Assume one internet-facing service is compromised and determine how far an +attacker can go. + +Checks: + +- Confirm container user IDs and whether services run as root. +- Confirm dropped and added Linux capabilities. +- Confirm whether any container has access to the Docker socket. +- Review writable host mounts and persistent volumes. +- Review whether secrets are exposed through environment variables, mounted + files, logs, or backups. +- Review inter-container network reachability and whether it is necessary. +- Review whether a compromise of tracker, Caddy, or Grafana can reach MySQL, + Prometheus, or sensitive files. +- Review cron jobs and backup scripts for privilege escalation or secret + exposure paths. + +## Phase 5: Supply-Chain Review + +Review whether the deployment depends on mutable, unpinned, or weakly verified +artifacts. + +Checks: + +- Confirm exact image digests for all deployed containers. +- Confirm whether images are pinned to immutable versions. +- Confirm whether any service is running from a mutable tag such as `develop` or + `latest`. +- Review recent CVEs affecting the deployed OS and container images. +- Review how deployment artifacts are produced and whether provenance can be + reconstructed. + +## Phase 6: Validation and Reporting + +After reviewing configuration and source, validate realistic attack paths in a +controlled and authorized manner. + +Validation principles: + +- Prefer non-destructive checks first. +- Avoid denial-of-service testing on production unless explicitly scheduled. +- Test the smallest proof needed to validate a finding. +- Record exact preconditions and evidence. + +Each finding should contain: + +- Title. +- Severity. +- Affected component. +- Preconditions. +- Attack path. +- Evidence. +- Impact. +- Recommended remediation. +- Whether the issue is configuration-specific, code-specific, or both. + +Reporting rules: + +- Record hypotheses and partial observations in the surface files while the + review is in progress. +- Move only confirmed findings into `findings.md`. +- Use `README.md` as the final review summary once the cycle is complete. +- Keep `progress.md` current so another reviewer can continue the work without + reconstructing context. + +## Recurring Review Checklist + +Use this checklist for each periodic review. + +- [ ] Reconfirm all public endpoints, ports, and DNS records. +- [ ] Reconfirm all Docker-published ports and container image digests. +- [ ] Reconfirm that only intended services are public. +- [ ] Reconfirm SSH hardening and user access. +- [ ] Review changes to Docker Compose, Caddy, tracker, firewall, and netplan. +- [ ] Review changes to secrets, tokens, backup scripts, and mounted volumes. +- [ ] Review tracker source changes affecting HTTP, UDP, API, auth, or proxy + handling. +- [ ] Review Grafana version, auth mode, plugins, and public dashboard settings. +- [ ] Review CVEs and upstream advisories for OS and deployed images. +- [ ] Re-evaluate any prior accepted risks. +- [ ] Re-run focused validation for all previously high-severity findings. +- [ ] Record new findings, changes in severity, and remediation status. + +## Evidence Request Template + +When preparing a review, gather the following from the live environment and any +upstream repositories involved: + +- Source repository URL and exact deployed revision for the tracker. +- Source repository URL and exact deployed revision for any custom backup, + startup, or helper service. +- Redacted `.env` file showing variable names. +- `docker ps` output. +- `docker inspect` output for each running container. +- `docker network inspect` output for each compose network. +- `ss -tulpn` output. +- `ufw status verbose` output. +- `sshd_config` and summary of local users with shell access. +- Grafana auth configuration and plugin list. +- OS package update status. + +## Known High-Priority Review Topics + +The following items should always receive explicit attention until their risk +is reduced or eliminated: + +- Mutable tracker image tag in Docker Compose. +- Public Grafana exposure. +- Public tracker API exposure. +- Trust in proxy headers for client IP attribution. +- Docker-published ports bypassing the host firewall model. +- Secret exposure through environment variables, mounted files, logs, or backup + archives. + +## Output of Each Review + +Each review cycle should produce a review folder containing: + +- `README.md` as the final summary report. +- `progress.md` as the live work tracker. +- `findings.md` as the confirmed findings register. +- One detailed note per reviewed service or attack surface. + +The final summary report should contain: + +- Review date. +- Reviewer. +- Deployment version summary. +- Scope covered in this cycle. +- Sources of evidence reviewed. +- Confirmed findings. +- Rejected hypotheses. +- Follow-up actions. +- Open questions requiring more information or source access. + +The detailed surface files should contain: + +- Scope of the surface under review. +- Hypotheses tested. +- Evidence reviewed. +- Checks performed. +- Findings or non-findings. +- Open questions and next actions. + +Store review reports under `docs/security/reviews/` and reference this plan from +each review-cycle `README.md`. diff --git a/docs/udp-conntrack-runbook.md b/docs/udp-conntrack-runbook.md new file mode 100644 index 0000000..5b27dec --- /dev/null +++ b/docs/udp-conntrack-runbook.md @@ -0,0 +1,338 @@ +<!-- cspell:ignore Rcvbuf conntrack softirq recvmmsg NoPorts ksoftirqd nproc vmstat mpstat --> + +# UDP Conntrack Runbook + +Operational guide for detecting, fixing, and explaining UDP packet loss caused +by conntrack saturation or related kernel-side packet-path pressure. + +This runbook exists for reuse beyond issue-specific evidence. For the incident +that led to the current tuning, see +[ISSUE-21](issues/ISSUE-21-scale-up-server-for-udp-uptime.md) and the evidence +under `docs/issues/evidence/ISSUE-21/`. + +## When To Use This Runbook + +Use this runbook when one or more of these symptoms appear: + +- newTrackon or other external probes show intermittent UDP timeouts +- UDP uptime drops while HTTP stays healthy +- UDP request volume is high and Docker DNAT is in the packet path +- `nf_conntrack` may be full or close to full +- Host load looks odd relative to per-CPU usage and packet drops are suspected + +## How To Detect The Problem + +### External Symptoms + +Common user-visible symptoms: + +- External UDP probes alternate between working and timing out +- Failures self-recover without a deploy or restart +- HTTP tracker remains mostly healthy while UDP uptime degrades +- Rolling uptime remains low for hours even after recent successful probes + +### Host Checks + +Run this on the live host: + +```bash +ssh demotracker ' + echo "=== conntrack counts ===" && + sudo sysctl net.netfilter.nf_conntrack_max net.netfilter.nf_conntrack_count && + echo "=== UDP timeouts ===" && + sudo sysctl net.netfilter.nf_conntrack_udp_timeout \ + net.netfilter.nf_conntrack_udp_timeout_stream && + echo "=== dmesg table full ===" && + sudo dmesg -T | grep -i "nf_conntrack: table full" | tail -10 && + echo "(no output = no table-full events)" && + echo "=== UDP receive errors ===" && + cat /proc/net/snmp | grep -E "^Udp:" | + awk "NR==1{for(i=1;i<=NF;i++) h[i]=\$i} NR==2{for(i=1;i<=NF;i++) print h[i]\": \"\$i}" | + grep -E "RcvbufErrors|InErrors|NoPorts" && + echo "=== UDP6 receive errors ===" && + cat /proc/net/snmp6 | grep -E "Udp6RcvbufErrors|Udp6InErrors|Udp6NoPorts" +' +``` + +Interpret the output like this: + +- `nf_conntrack_count == nf_conntrack_max`: immediate problem; table is full +- `dmesg` contains `nf_conntrack: table full, dropping packet`: confirmed drops +- `UdpRcvbufErrors > 0` or `Udp6RcvbufErrors > 0`: receive-buffer drops exist +- `UdpNoPorts` or `Udp6NoPorts`: usually benign; probes to closed ports, not the tracker itself + +### Optional Load Distribution Check + +Use this when load average looks high but per-process CPU usage does not explain +it clearly: + +```bash +ssh demotracker ' + uptime && + nproc && + mpstat -P ALL 1 1 2>/dev/null || echo "mpstat not available" && + ps -eo pid,comm,%cpu,%mem,stat --sort=-%cpu | head -15 && + vmstat 1 3 +' +``` + +Interpretation: + +- high `%soft` on one CPU means kernel packet handling is concentrated there +- this points to softirq/RX steering imbalance, not necessarily tracker code problems +- this is a separate bottleneck from conntrack table saturation + +## How To Fix It + +### Immediate Live Fix + +Apply the kernel tuning live: + +```bash +ssh demotracker ' + sudo sysctl -w net.netfilter.nf_conntrack_max=1048576 && + sudo sysctl -w net.netfilter.nf_conntrack_udp_timeout=10 && + sudo sysctl -w net.netfilter.nf_conntrack_udp_timeout_stream=15 +' +``` + +### Persist The Fix In This Repository + +The persistent configuration lives in: + +- `server/etc/sysctl.d/99-conntrack.conf` +- `server/etc/modules-load.d/conntrack.conf` + +Why both files matter: + +- `99-conntrack.conf` stores the kernel parameter values +- `conntrack.conf` preloads the `nf_conntrack` module at boot +- without preloading, the `net.netfilter.*` keys may not exist yet when systemd applies sysctl files, so the values can be skipped after reboot + +Current tuned values used by this repository: + +| Key | Value | +| ----------------------------------------------- | --------- | +| `net.netfilter.nf_conntrack_max` | `1048576` | +| `net.netfilter.nf_conntrack_udp_timeout` | `10` | +| `net.netfilter.nf_conntrack_udp_timeout_stream` | `15` | + +### Validate After The Change + +Re-run the detection command above and confirm all of these: + +- `nf_conntrack_count` is well below `nf_conntrack_max` +- no fresh `table full` messages appear in `dmesg` +- `UdpRcvbufErrors` and `Udp6RcvbufErrors` are stable or zero +- external UDP probes recover and remain healthy for multiple hours or days + +## Why This Works + +### Packet Path + +For the deployed tracker, the UDP receive path is approximately: + +```text +NIC -> kernel RX interrupt -> softirq/ksoftirqd -> conntrack + Docker DNAT -> socket buffer -> tracker recv loop -> spawned async task +``` + +The important point is that conntrack lookup and DNAT happen in the kernel +before the tracker reads the packet from the socket. + +### Failure Mechanism + +With Docker in the packet path, each UDP packet can create or refresh a +conntrack entry. + +If all of these are true at the same time: + +- request rate is high +- `nf_conntrack_max` is too small +- UDP entry timeouts are too long + +then the steady-state number of tracked UDP flows grows until the table is full. +Once full, the kernel drops new packets before the tracker can read them. + +### Why Increasing `nf_conntrack_max` Helps + +Increasing `nf_conntrack_max` raises the ceiling for concurrent tracked flows, +reducing the chance that bursts or sustained load fill the table. + +### Why Reducing UDP Timeouts Helps + +Reducing `nf_conntrack_udp_timeout` and +`nf_conntrack_udp_timeout_stream` shortens how long old UDP entries stay in the +table. + +That reduces steady-state occupancy, which is often more important than raw CPU +capacity for this failure mode. + +### Why The Tracker Code Is Not The Root Cause + +The tracker's UDP loop reads packets after the kernel has already: + +- handled the RX interrupt/softirq work +- performed conntrack lookup +- applied Docker NAT rules +- copied the packet into the socket receive buffer + +If packets are being dropped because the conntrack table is full, the tracker +never sees them. + +## Separate Future Tuning: RPS/RFS + +RPS and RFS are not part of the current deployed fix. They address a different +bottleneck: one CPU being saturated by kernel softirq work while other CPUs sit +idle. They solve a different problem from conntrack table saturation. + +### How To Detect The Need For RPS/RFS + +Run the load distribution check: + +```bash +ssh demotracker ' + uptime && + nproc && + mpstat -P ALL 1 1 && + vmstat 1 3 +' +``` + +Signals that RPS/RFS may help: + +- one CPU shows `%soft` near or above 80–90% while other CPUs have significant + `%idle` +- `vmstat` shows very high interrupt counts (`in` column) and many context + switches (`cs` column) +- `ps` shows `ksoftirqd/<N>` for a single CPU near the top of CPU consumers + +This pattern was first observed on this server on 2026-04-27: + +```text +CPU2: %usr=4.76 %sys=4.76 %soft=80.95 %idle=9.52 +``` + +All other CPUs were at approximately 44% idle at the same time. + +### Why It Happens + +The Hetzner VM's virtio-net NIC has a single RX queue. Linux assigns that +queue's hardware interrupt to one CPU. All softirq processing for every incoming +packet — UDP and TCP — flows through that one core. + +The softirq work includes: + +- NIC DMA and descriptor processing +- IP and UDP checksums +- conntrack lookup and Docker DNAT +- socket demux and receive-buffer copy + +Until one of the RX-steering features is enabled, the kernel has no way to +distribute this work. + +### How To Estimate Whether This Is A Real Bottleneck + +At current peak UDP traffic of ~750 req/s and HTTP of ~2000 req/s, the softirq +CPU (CPU2) was at about 81%. Saturation would occur if that figure approaches +100% consistently. + +A rough rule of thumb: if total req/s grows by roughly 2.5× from the 2026-04-27 +baseline (~2750 req/s combined) without any RPS/RFS tuning, CPU2 may saturate +and become the next source of packet loss. + +### How To Fix It — RPS + +RPS tells the kernel to re-queue softirq processing for each packet onto a +different CPU, chosen by hashing the packet's 4-tuple (src IP, src port, dst +IP, dst port). + +Check the NIC and queue name first: + +```bash +ssh demotracker 'ls /sys/class/net/' +ssh demotracker 'ls /sys/class/net/eth0/queues/' +``` + +Enable RPS across all 8 CPUs: + +```bash +ssh demotracker 'echo ff | sudo tee /sys/class/net/eth0/queues/rx-0/rps_cpus' +``` + +The value `ff` is a bitmask: `0xff` = all 8 CPUs. Adjust for the actual CPU +count if the server is resized. + +### How To Fix It — RFS + +RFS extends RPS by tracking which CPU most recently ran the application socket +thread and steers softirq toward that CPU. This reduces cache misses when the +kernel hands the packet to userspace. + +Enable RFS: + +```bash +ssh demotracker ' + sudo sysctl -w net.core.rps_sock_flow_entries=32768 && + echo 4096 | sudo tee /sys/class/net/eth0/queues/rx-0/rps_flow_cnt +' +``` + +### Making RPS/RFS Persistent + +The `/sys/class/net/...` paths do not survive reboot. To persist them, add a +`systemd` service or a `@reboot` cron entry, and record the kernel parameter in +`/etc/sysctl.d/`. + +Example cron entry (`/etc/cron.d/rps`): + +```text +@reboot root echo ff > /sys/class/net/eth0/queues/rx-0/rps_cpus && echo 4096 > /sys/class/net/eth0/queues/rx-0/rps_flow_cnt +``` + +If this is deployed permanently, add the config to: + +- `server/etc/sysctl.d/` for the `net.core.rps_sock_flow_entries` setting +- `server/etc/cron.d/` for the sysfs writes + +### Validate After Enabling + +Re-run `mpstat -P ALL 1 5` and confirm that: + +- `%soft` is spread across multiple CPUs instead of concentrated on one +- the formerly saturated CPU drops below 70–80% +- external UDP uptime remains stable or improves + +### Why RPS/RFS Does Not Break Conntrack + +RPS reorders which CPU handles softirq, but does not bypass conntrack or DNAT. +Each packet still goes through the full kernel stack; it just does so on a +different CPU. The conntrack settings from `99-conntrack.conf` remain in effect +independently. + +### Why This Is Separate From The Conntrack Fix + +Conntrack overflow causes packet **drops** — the kernel silently discards the +packet before it ever enters a socket buffer. RPS/RFS addresses CPU **hotspot** +— one core being too busy to process incoming packets fast enough. Both can +cause UDP timeouts, but the diagnostic signals are different and the fixes do +not overlap. + +## Reference Values From The 2026-04-27 Verification + +Recorded before merging PR #22: + +- peak UDP tracker traffic observed over the prior 7 days: about `750 req/s` +- peak HTTP tracker traffic observed over the prior 7 days: about `2000 req/s` +- `nf_conntrack_count`: `341652` +- `nf_conntrack_max`: `1048576` +- utilization: `32.59%` +- `UdpRcvbufErrors`: `0` +- `Udp6RcvbufErrors`: `56` cumulative since boot, not material at observed load + +## Related Files + +- [docs/infrastructure.md](infrastructure.md) +- [docs/infrastructure-resize-history.md](infrastructure-resize-history.md) +- [server/etc/sysctl.d/99-conntrack.conf](../server/etc/sysctl.d/99-conntrack.conf) +- [server/etc/modules-load.d/conntrack.conf](../server/etc/modules-load.d/conntrack.conf) +- [docs/issues/ISSUE-21-scale-up-server-for-udp-uptime.md](issues/ISSUE-21-scale-up-server-for-udp-uptime.md) diff --git a/project-words.txt b/project-words.txt index 9640700..246e421 100644 --- a/project-words.txt +++ b/project-words.txt @@ -4,6 +4,8 @@ Brevo Caddyfile DNAT EADDRINUSE +EACCES +ERRO Freshping Gatus Leechers @@ -20,17 +22,24 @@ Vigil agentskills augmentedcode behaviour +bencoded +bleve bindv6only +bitmask clippy codel conntrack +cpus crontab ctstate demotracker deogmiudufm dockerized +defence +drilldown dport dtolnay +hotspot efivarfs efivars ethernets @@ -38,34 +47,63 @@ frontmatter fswc grafana healthcheck +ksoftirqd leecher logfile misrouted mkpath +Mailgun +mpstat mysqladmin netnsid +netfilter netplan networkd newtrackon noall noprefixroute noqueue +oneline +organised oneshot pkts post-mortems prometheus +pyroscope +datagrams +demux +HSTS +nosniff +parseable +pipefail qdisc qlen repomix +realised rgba +runbook rustfmt shellcheck +snmp sourceable +signup +sysfs tcpdump tera timepicker tmpfs torrust +tulpn ulnp +usagestats userland +userspace veth +virtio +runqueue +overcommitted +cutover +Mbit +CPUPerc +Celano +urlencode diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh new file mode 100755 index 0000000..34f9ed8 --- /dev/null +++ b/scripts/pre-commit.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +"$script_dir/lint.sh" diff --git a/server/etc/cron.d/rps-rfs b/server/etc/cron.d/rps-rfs new file mode 100644 index 0000000..0429a9a --- /dev/null +++ b/server/etc/cron.d/rps-rfs @@ -0,0 +1,7 @@ +# Persist RPS/RFS sysfs settings across reboot. +# See: docs/udp-conntrack-runbook.md and ISSUE-29. + +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +@reboot root echo ff > /sys/class/net/eth0/queues/rx-0/rps_cpus && echo 4096 > /sys/class/net/eth0/queues/rx-0/rps_flow_cnt diff --git a/server/etc/modules-load.d/conntrack.conf b/server/etc/modules-load.d/conntrack.conf new file mode 100644 index 0000000..ca3f9f2 --- /dev/null +++ b/server/etc/modules-load.d/conntrack.conf @@ -0,0 +1,7 @@ +# Pre-load nf_conntrack so that net.netfilter.* sysctl settings in +# /etc/sysctl.d/99-conntrack.conf are applied at boot. +# +# Without this, systemd applies sysctl configs before Docker loads nf_conntrack, +# so the net.netfilter.* keys do not exist yet and are silently skipped. +# See: https://github.com/torrust/torrust-tracker-demo/issues/21 +nf_conntrack diff --git a/server/etc/sysctl.d/98-rps-rfs.conf b/server/etc/sysctl.d/98-rps-rfs.conf new file mode 100644 index 0000000..8e0cc05 --- /dev/null +++ b/server/etc/sysctl.d/98-rps-rfs.conf @@ -0,0 +1,5 @@ +# RPS/RFS tuning to distribute NIC RX softirq work across CPUs. +# See: docs/udp-conntrack-runbook.md and ISSUE-29. + +# Global socket flow table size used by RFS. +net.core.rps_sock_flow_entries = 32768 diff --git a/server/etc/sysctl.d/99-conntrack.conf b/server/etc/sysctl.d/99-conntrack.conf new file mode 100644 index 0000000..b50758b --- /dev/null +++ b/server/etc/sysctl.d/99-conntrack.conf @@ -0,0 +1,22 @@ +# Kernel tuning for UDP tracker running behind Docker bridge networking. +# Docker DNAT creates a conntrack entry for every packet. Under high UDP tracker +# load the defaults cause silent packet drops and intermittent timeouts. +# See: https://github.com/torrust/torrust-tracker-demo/issues/21 +# +# NOTE: net.netfilter.* settings are silently skipped at boot if the +# nf_conntrack module is not yet loaded. Pre-load it via: +# /etc/modules-load.d/conntrack.conf + +# Maximum conntrack table entries. +# Default: 65536-262144. At 400 UDP req/s with a 120 s stream timeout the +# table fills (400 * 120 = 48000 entries minimum). At 1500 req/s it overflows. +# Each entry uses ~300 bytes; 1 M entries ≈ 300 MB. +net.netfilter.nf_conntrack_max = 1048576 + +# UDP stream timeout (bidirectional). Default: 120 s. +# A tracker request-reply completes in milliseconds; 15 s is generous. +# Reducing from 120 s cuts steady-state table size by ~8x. +net.netfilter.nf_conntrack_udp_timeout_stream = 15 + +# UDP single-direction timeout. Default: 30 s. +net.netfilter.nf_conntrack_udp_timeout = 10 diff --git a/server/opt/torrust/.env b/server/opt/torrust/.env index 9fb6668..0612aad 100644 --- a/server/opt/torrust/.env +++ b/server/opt/torrust/.env @@ -52,3 +52,5 @@ MYSQL_PASSWORD='<MYSQL_PASSWORD>' # WARNING: Change default credentials in production deployments for security GF_SECURITY_ADMIN_USER='admin' GF_SECURITY_ADMIN_PASSWORD='<GF_SECURITY_ADMIN_PASSWORD>' +# Grafana server root URL — used to generate correct public dashboard share links +GF_SERVER_ROOT_URL='https://grafana.torrust-tracker-demo.com' diff --git a/server/opt/torrust/docker-compose.yml b/server/opt/torrust/docker-compose.yml index bb54f8e..1cbc9e5 100644 --- a/server/opt/torrust/docker-compose.yml +++ b/server/opt/torrust/docker-compose.yml @@ -51,7 +51,7 @@ services: # Placed first as it's the entry point for HTTPS traffic caddy: <<: *defaults - image: caddy:2.10 + image: caddy:2.10.2 container_name: caddy # NOTE: No UFW firewall rule needed for these ports! # Docker-published ports bypass iptables/UFW rules entirely. @@ -63,7 +63,7 @@ services: - "80:80" # HTTPS - "443:443" - # HTTP/3 (QUIC) + # HTTP/3 (QUIC) re-enabled in ISSUE-31 - "443:443/udp" volumes: - ./storage/caddy/etc/Caddyfile:/etc/caddy/Caddyfile:ro @@ -110,7 +110,7 @@ services: prometheus: <<: *defaults - image: prom/prometheus:v3.5.0 + image: prom/prometheus:v3.5.1 container_name: prometheus networks: - metrics_network @@ -133,7 +133,7 @@ services: grafana: <<: *defaults - image: grafana/grafana:12.3.1 + image: grafana/grafana:12.4.2 container_name: grafana networks: - visualization_network @@ -141,6 +141,7 @@ services: environment: - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER} - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} + - GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL} volumes: - ./storage/grafana/data:/var/lib/grafana - ./storage/grafana/provisioning:/etc/grafana/provisioning:ro diff --git a/server/opt/torrust/storage/grafana/provisioning/dashboards/torrust/stats.json b/server/opt/torrust/storage/grafana/provisioning/dashboards/torrust/stats.json deleted file mode 100644 index c53ea60..0000000 --- a/server/opt/torrust/storage/grafana/provisioning/dashboards/torrust/stats.json +++ /dev/null @@ -1,1420 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Using stats endpoint:\n\nhttps://tracker.example.com/api/v1/stats?token=MyAccessToken&format=prometheus", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "completed{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Completed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "torrents{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Torrents", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "seeders{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Seeders", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "leechers{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Leechers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Request per second in 15 minutes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 0, - "y": 5 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(udp4_connections_handled{job=\"tracker_stats\"}[15m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP4 Connections (per sec)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Request per second in 15 minutes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 6, - "y": 5 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(udp4_announces_handled[15m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP4 Announces (per sec)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Request per second in 15 minutes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 12, - "y": 5 - }, - "id": 21, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(udp4_scrapes_handled[15m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP4 Scrapes (per sec)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Request per second in 15 minutes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 18, - "y": 5 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(udp4_errors_handled[15m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP4 Errors (per sec)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "UDP Average Connect Processing Time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ns" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 14 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "udp_avg_connect_processing_time_ns{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP Average Connect Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "UDP Average Announce Processing Time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ns" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 14 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "udp_avg_announce_processing_time_ns{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP Average Announce Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "UDP Average Scrape Processing Time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ns" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 14 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "udp_avg_scrape_processing_time_ns{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP Average Scrape Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "UDP banned requests (per second)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 14 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(udp_requests_banned{job=\"tracker_stats\"}[15m])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP banned requests (per second)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "UDP 4 requests and responses", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 18, - "x": 0, - "y": 19 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(udp4_requests{job=\"tracker_stats\"}[15m])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(udp4_responses[15m])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "UDP4 requests and responses (per second)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "UDP Banned IPs", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 19 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "udp_banned_ips_total{job=\"tracker_stats\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP Banned IPs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 25 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(udp_requests_aborted{job=\"tracker_stats\"}[15m])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "UDP aborted requests (per second)", - "type": "timeseries" - } - ], - "preload": false, - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Torrust Live Demo Tracker (stats)", - "uid": "de6lx6hce8fswc", - "version": 93, - "weekStart": "" -} \ No newline at end of file