diff --git a/Cargo.lock b/Cargo.lock index 328e2db93..6def147d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,6 +484,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.21.7" @@ -560,8 +569,11 @@ dependencies = [ "bittorrent-primitives", "bittorrent-tracker-core", "criterion", + "formatjson", "futures", "mockall", + "serde", + "serde_json", "thiserror 2.0.12", "tokio", "torrust-tracker-configuration", @@ -1558,6 +1570,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "formatjson" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3ba17cfe2aff8969f35b2bffec13b34756c51ea53eadcc5d5446f71370e2ed" +dependencies = [ + "miette", + "thiserror 1.0.69", +] + [[package]] name = "forwarded-header-value" version = "0.1.1" @@ -2295,6 +2317,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2484,6 +2512,37 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miette" +version = "7.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a955165f87b37fd1862df2a59547ac542c77ef6d17c666f619d1ad22dd89484" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "thiserror 1.0.69", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf45bf44ab49be92fd1227a3be6fc6f617f1a337c06af54981048574d8783147" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "mime" version = "0.3.17" @@ -2815,6 +2874,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" + [[package]] name = "parking" version = "2.2.1" @@ -3976,6 +4041,27 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f44ed3c63152de6a9f90acbea1a110441de43006ea51bcce8f436196a288b" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" version = "1.0.109" @@ -4084,6 +4170,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" +dependencies = [ + "rustix 1.0.3", + "windows-sys 0.59.0", +] + [[package]] name = "termtree" version = "0.5.1" @@ -4119,6 +4215,16 @@ dependencies = [ "url", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "unicode-linebreak", + "unicode-width 0.2.0", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4862,6 +4968,24 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" diff --git a/cSpell.json b/cSpell.json index 3121d6175..31594b797 100644 --- a/cSpell.json +++ b/cSpell.json @@ -59,6 +59,7 @@ "Eray", "filesd", "flamegraph", + "formatjson", "Freebox", "Frostegård", "gecos", diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs index 484c12ff9..0c4712506 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs @@ -9,9 +9,9 @@ use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepo use bittorrent_udp_tracker_core::services::banning::BanService; use serde::Deserialize; use tokio::sync::RwLock; -use torrust_rest_tracker_api_core::statistics::services::get_metrics; +use torrust_rest_tracker_api_core::statistics::services::{get_labeled_metrics, get_metrics}; -use super::responses::{metrics_response, stats_response}; +use super::responses::{labeled_stats_response, metrics_response, stats_response}; #[derive(Deserialize, Debug, Default)] #[serde(rename_all = "lowercase")] @@ -57,3 +57,24 @@ pub async fn get_stats_handler( None => stats_response(metrics), } } + +#[allow(clippy::type_complexity)] +pub async fn get_metrics_handler( + State(state): State<( + Arc, + Arc>, + Arc, + Arc, + )>, + params: Query, +) -> Response { + let metrics = get_labeled_metrics(state.0.clone(), state.1.clone(), state.2.clone(), state.3.clone()).await; + + match params.0.format { + Some(format) => match format { + Format::Json => labeled_stats_response(metrics), + Format::Prometheus => todo!(), + }, + None => labeled_stats_response(metrics), + } +} diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs index d9480259e..c7e905708 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs @@ -1,7 +1,8 @@ //! API resources for the [`stats`](crate::v1::context::stats) //! API context. +use bittorrent_http_tracker_core::statistics::metrics::LabeledMetric; use serde::{Deserialize, Serialize}; -use torrust_rest_tracker_api_core::statistics::services::TrackerMetrics; +use torrust_rest_tracker_api_core::statistics::services::{TrackerLabeledMetrics, TrackerMetrics}; /// It contains all the statistics generated by the tracker. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -116,6 +117,22 @@ impl From for Stats { } } +/// It contains all the statistics generated by the tracker. +#[derive(Serialize, Debug, PartialEq, Eq)] +pub struct LabeledStats { + // Extendable metrics + labeled_metrics: Vec, +} + +impl From for LabeledStats { + #[allow(deprecated)] + fn from(metrics: TrackerLabeledMetrics) -> Self { + Self { + labeled_metrics: metrics.labeled_metrics, + } + } +} + #[cfg(test)] mod tests { use torrust_rest_tracker_api_core::statistics::metrics::Metrics; diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/responses.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/responses.rs index 853fdd2e2..76f0bad10 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/responses.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/responses.rs @@ -1,9 +1,9 @@ //! API responses for the [`stats`](crate::v1::context::stats) //! API context. use axum::response::{IntoResponse, Json, Response}; -use torrust_rest_tracker_api_core::statistics::services::TrackerMetrics; +use torrust_rest_tracker_api_core::statistics::services::{TrackerLabeledMetrics, TrackerMetrics}; -use super::resources::Stats; +use super::resources::{LabeledStats, Stats}; /// `200` response that contains the [`Stats`] resource as json. #[must_use] @@ -11,6 +11,12 @@ pub fn stats_response(tracker_metrics: TrackerMetrics) -> Response { Json(Stats::from(tracker_metrics)).into_response() } +/// `200` response that contains the [`LabeledStats`] resource as json. +#[must_use] +pub fn labeled_stats_response(tracker_metrics: TrackerLabeledMetrics) -> Response { + Json(LabeledStats::from(tracker_metrics)).into_response() +} + /// `200` response that contains the [`Stats`] resource in Prometheus Text Exposition Format . #[allow(deprecated)] #[must_use] diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs index e92b5b34d..d516e5ffb 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs @@ -9,17 +9,27 @@ use axum::routing::get; use axum::Router; use torrust_rest_tracker_api_core::container::TrackerHttpApiCoreContainer; -use super::handlers::get_stats_handler; +use super::handlers::{get_metrics_handler, get_stats_handler}; /// It adds the routes to the router for the [`stats`](crate::v1::context::stats) API context. pub fn add(prefix: &str, router: Router, http_api_container: &Arc) -> Router { - router.route( - &format!("{prefix}/stats"), - get(get_stats_handler).with_state(( - http_api_container.tracker_core_container.in_memory_torrent_repository.clone(), - http_api_container.ban_service.clone(), - http_api_container.http_stats_repository.clone(), - http_api_container.udp_server_stats_repository.clone(), - )), - ) + router + .route( + &format!("{prefix}/stats"), + get(get_stats_handler).with_state(( + http_api_container.tracker_core_container.in_memory_torrent_repository.clone(), + http_api_container.ban_service.clone(), + http_api_container.http_stats_repository.clone(), + http_api_container.udp_server_stats_repository.clone(), + )), + ) + .route( + &format!("{prefix}/metrics"), + get(get_metrics_handler).with_state(( + http_api_container.tracker_core_container.in_memory_torrent_repository.clone(), + http_api_container.ban_service.clone(), + http_api_container.http_stats_repository.clone(), + http_api_container.udp_server_stats_repository.clone(), + )), + ) } diff --git a/packages/http-tracker-core/Cargo.toml b/packages/http-tracker-core/Cargo.toml index aaf982b04..49ef52881 100644 --- a/packages/http-tracker-core/Cargo.toml +++ b/packages/http-tracker-core/Cargo.toml @@ -20,6 +20,7 @@ bittorrent-primitives = "0.1.0" bittorrent-tracker-core = { version = "3.0.0-develop", path = "../tracker-core" } criterion = { version = "0.5.1", features = ["async_tokio"] } futures = "0" +serde = "1.0.219" thiserror = "2" tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal", "sync"] } torrust-tracker-configuration = { version = "3.0.0-develop", path = "../configuration" } @@ -27,7 +28,9 @@ torrust-tracker-primitives = { version = "3.0.0-develop", path = "../primitives" tracing = "0" [dev-dependencies] +formatjson = "0.3.1" mockall = "0" +serde_json = "1.0.140" torrust-tracker-test-helpers = { version = "3.0.0-develop", path = "../test-helpers" } [[bench]] diff --git a/packages/http-tracker-core/src/statistics/event/handler.rs b/packages/http-tracker-core/src/statistics/event/handler.rs index 0df1c41d3..7d60d3f9f 100644 --- a/packages/http-tracker-core/src/statistics/event/handler.rs +++ b/packages/http-tracker-core/src/statistics/event/handler.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::net::IpAddr; use crate::event::Event; @@ -9,22 +10,66 @@ use crate::statistics::repository::Repository; /// version of the event. pub async fn handle_event(event: Event, stats_repository: &Repository) { match event { - Event::TcpAnnounce { connection } => match connection.client_ip_addr() { - IpAddr::V4(_) => { - stats_repository.increase_tcp4_announces().await; + Event::TcpAnnounce { connection } => { + // Global fixed metrics + + match connection.client_ip_addr() { + IpAddr::V4(_) => { + stats_repository.increase_tcp4_announces().await; + } + IpAddr::V6(_) => { + stats_repository.increase_tcp6_announces().await; + } } - IpAddr::V6(_) => { - stats_repository.increase_tcp6_announces().await; - } - }, - Event::TcpScrape { connection } => match connection.client_ip_addr() { - IpAddr::V4(_) => { - stats_repository.increase_tcp4_scrapes().await; - } - IpAddr::V6(_) => { - stats_repository.increase_tcp6_scrapes().await; + + // Extendable metrics + + let ip_version = match connection.client_ip_addr() { + IpAddr::V4(_) => "ipv4".to_string(), + IpAddr::V6(_) => "ipv6".to_string(), + }; + + stats_repository + .increase_counter( + "announce_requests_received_total", + &BTreeMap::from([ + ("ip_version".to_string(), ip_version), + ("protocol".to_string(), "http".to_string()), + ("url".to_string(), format!("http://{}", connection.server_socket_addr())), // todo: use the actual scheme + ]), + ) + .await; + } + Event::TcpScrape { connection } => { + // Global fixed metrics + + match connection.client_ip_addr() { + IpAddr::V4(_) => { + stats_repository.increase_tcp4_scrapes().await; + } + IpAddr::V6(_) => { + stats_repository.increase_tcp6_scrapes().await; + } } - }, + + // Extendable metrics + + let ip_version = match connection.client_ip_addr() { + IpAddr::V4(_) => "ipv4".to_string(), + IpAddr::V6(_) => "ipv6".to_string(), + }; + + stats_repository + .increase_counter( + "scrape_requests_received_total", + &BTreeMap::from([ + ("ip_version".to_string(), ip_version), + ("protocol".to_string(), "http".to_string()), + ("url".to_string(), format!("http://{}", connection.server_socket_addr())), // todo: use the actual scheme + ]), + ) + .await; + } } tracing::debug!("stats: {:?}", stats_repository.get_stats().await); diff --git a/packages/http-tracker-core/src/statistics/metrics.rs b/packages/http-tracker-core/src/statistics/metrics.rs index 6c102770b..db669324e 100644 --- a/packages/http-tracker-core/src/statistics/metrics.rs +++ b/packages/http-tracker-core/src/statistics/metrics.rs @@ -1,12 +1,9 @@ +use std::collections::BTreeMap; + +use serde::Serialize; + /// Metrics collected by the tracker. -/// -/// - Number of connections handled -/// - Number of `announce` requests handled -/// - Number of `scrape` request handled -/// -/// These metrics are collected for each connection type: UDP and HTTP -/// and also for each IP version used by the peers: IPv4 and IPv6. -#[derive(Debug, PartialEq, Default)] +#[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct Metrics { /// Total number of TCP (HTTP tracker) `announce` requests from IPv4 peers. pub tcp4_announces_handled: u64, @@ -19,4 +16,108 @@ pub struct Metrics { /// Total number of TCP (HTTP tracker) `scrape` requests from IPv6 peers. pub tcp6_scrapes_handled: u64, + + pub labeled_metrics: Vec, +} + +impl Metrics { + pub fn increase_counter(&mut self, metric_name: &str, metric_labels: &BTreeMap) { + let mut found = false; + + for labeled_metric in &mut self.labeled_metrics { + // todo: + // - Check that the metric has the counter type. + + if labeled_metric.metric.name == metric_name && labeled_metric.labels == *metric_labels { + labeled_metric.metric.value += 1; + found = true; + break; + } + } + + if !found { + self.labeled_metrics.push(LabeledMetric { + metric: Metric { + name: metric_name.to_string(), + kind: "counter".to_string(), + value: 1, + }, + labels: metric_labels.clone(), + }); + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize)] +pub struct LabeledMetric { + pub metric: Metric, + pub labels: BTreeMap, +} + +#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize)] +pub struct Metric { + pub name: String, + pub kind: String, + pub value: u64, // todo: change to f64. See https://prometheus.io/docs/concepts/data_model/#samples +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::LabeledMetric; + use crate::statistics::metrics::{Metric, Metrics}; + + #[allow(clippy::no_effect_replace)] + #[test] + fn metrics_should_be_serializable_to_json() { + let metrics = Metrics { + tcp4_announces_handled: 1, + tcp4_scrapes_handled: 2, + tcp6_announces_handled: 3, + tcp6_scrapes_handled: 4, + labeled_metrics: vec![LabeledMetric { + metric: Metric { + name: "announce_requests_received_total".to_string(), + kind: "counter".to_string(), + value: 325, + }, + labels: BTreeMap::from([ + ("ip_version".to_string(), "ipv4".to_string()), + ("protocol".to_string(), "udp".to_string()), + ("url".to_string(), "udp://127.0.0.1:6969".to_string()), + ]), + }], + }; + + let json = serde_json::to_string(&metrics).unwrap(); + + assert_eq!( + formatjson::format_json(&json).unwrap(), + formatjson::format_json( + r#" + { + "tcp4_announces_handled":1, + "tcp4_scrapes_handled":2, + "tcp6_announces_handled":3, + "tcp6_scrapes_handled":4, + "labeled_metrics": [ + { + "metric": { + "name": "announce_requests_received_total", + "kind": "counter", + "value": 325 + }, + "labels": { + "ip_version":"ipv4", + "protocol":"udp", + "url":"udp://127.0.0.1:6969" + } + } + ] + }"# + ) + .unwrap() + ); + } } diff --git a/packages/http-tracker-core/src/statistics/repository.rs b/packages/http-tracker-core/src/statistics/repository.rs index 5e15fc298..7ba91ce31 100644 --- a/packages/http-tracker-core/src/statistics/repository.rs +++ b/packages/http-tracker-core/src/statistics/repository.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::{RwLock, RwLockReadGuard}; @@ -51,4 +52,10 @@ impl Repository { stats_lock.tcp6_scrapes_handled += 1; drop(stats_lock); } + + pub async fn increase_counter(&self, metric_name: &str, metric_labels: &BTreeMap) { + let mut stats_lock = self.stats.write().await; + stats_lock.increase_counter(metric_name, metric_labels); + drop(stats_lock); + } } diff --git a/packages/http-tracker-core/src/statistics/services.rs b/packages/http-tracker-core/src/statistics/services.rs index dce7098b9..ab355790f 100644 --- a/packages/http-tracker-core/src/statistics/services.rs +++ b/packages/http-tracker-core/src/statistics/services.rs @@ -59,6 +59,8 @@ pub async fn get_metrics( // TCPv6 tcp6_announces_handled: stats.tcp6_announces_handled, tcp6_scrapes_handled: stats.tcp6_scrapes_handled, + // Samples + labeled_metrics: stats.labeled_metrics.clone(), }, } } diff --git a/packages/rest-tracker-api-core/src/statistics/services.rs b/packages/rest-tracker-api-core/src/statistics/services.rs index c40f7c82e..d14051850 100644 --- a/packages/rest-tracker-api-core/src/statistics/services.rs +++ b/packages/rest-tracker-api-core/src/statistics/services.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use bittorrent_http_tracker_core::statistics::metrics::LabeledMetric; use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository; use bittorrent_udp_tracker_core::services::banning::BanService; use bittorrent_udp_tracker_core::{self}; @@ -77,6 +78,31 @@ pub async fn get_metrics( } } +#[derive(Debug, PartialEq)] +pub struct TrackerLabeledMetrics { + // Extendable metrics + pub labeled_metrics: Vec, +} + +/// It returns all the [`TrackerLabeledMetrics`] +#[allow(deprecated)] +pub async fn get_labeled_metrics( + in_memory_torrent_repository: Arc, + ban_service: Arc>, + http_stats_repository: Arc, + udp_server_stats_repository: Arc, +) -> TrackerLabeledMetrics { + let _torrents_metrics = in_memory_torrent_repository.get_torrents_metrics(); + let _udp_banned_ips_total = ban_service.read().await.get_banned_ips_total(); + let _udp_server_stats = udp_server_stats_repository.get_stats().await; + + let http_stats = http_stats_repository.get_stats().await; + + TrackerLabeledMetrics { + labeled_metrics: http_stats.labeled_metrics.clone(), + } +} + #[cfg(test)] mod tests { use std::sync::Arc;