From 9d0714db67db0b17610f525516f2ff5d9570c256 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 14:04:15 +0200 Subject: [PATCH 1/8] Updating README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 929585c11..bb4649271 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Torrust Tracker is a lightweight but incredibly powerful and feature-rich BitTor * [X] Peer authentication using time-bound keys * [X] newTrackon check supported for both HTTP, UDP, where IPv4 and IPv6 is properly handled * [X] SQLite3 Persistent loading and saving of the torrent hashes and completed count +* [X] MySQL support added as engine option +* [X] Periodically saving added, interval can be configured ### Implemented BEPs * [BEP 3](https://www.bittorrent.org/beps/bep_0003.html): The BitTorrent Protocol From 9bc1ebb6057e94b7c70e4571f6e4ab67da6c142c Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 15:50:59 +0200 Subject: [PATCH 2/8] Fixing possible bug --- src/tracker.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tracker.rs b/src/tracker.rs index 0e42f69e1..a67cb62e6 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use log::info; +use log::{debug, info}; use serde::{Deserialize, Serialize}; use serde; use tokio::sync::{RwLock, RwLockReadGuard}; @@ -128,6 +128,7 @@ impl TorrentTracker { let torrents = self.database.load_persistent_torrent_data().await?; for torrent in torrents { + debug!("{:#?}", torrent); let _ = self.add_torrent(torrent.0, 0, torrent.1, 0).await; } @@ -308,7 +309,7 @@ impl TorrentTracker { let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); for (k, completed) in updates.iter() { - updates_cloned.insert(*k, *completed); + updates_cloned.insert(k.clone(), completed.clone()); } updates.clear(); drop(updates); @@ -318,16 +319,16 @@ impl TorrentTracker { if shadows.contains_key(k) { shadows.remove(k); } - shadows.insert(*k, *completed); + shadows.insert(k.clone(), completed.clone()); } drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. let mut shadow_copy: BTreeMap = BTreeMap::new(); for (infohash, completed) in shadows.iter() { - shadow_copy.insert(*infohash, TorrentEntry { + shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), - completed: *completed, + completed: completed.clone(), seeders: 0, }); } From 0c332cf382cbc26c325cd79e557b894e3aa57d9c Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 16:24:04 +0200 Subject: [PATCH 3/8] Fixing a buggy SQLite3 and MySQL --- src/mysql_database.rs | 4 ++-- src/sqlite_database.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index 7ecae214a..eea2190a3 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -82,10 +82,10 @@ impl Database for MysqlDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = completed", (info_hash.to_string(), completed.to_string())).is_err() { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", (info_hash.to_string(), completed.to_string())).is_err() { return Err(Error::InvalidQuery); } - debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = completed", info_hash.to_string(), completed.to_string()); + debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", info_hash.to_string(), completed.to_string()); } if db_transaction.commit().is_err() { diff --git a/src/sqlite_database.rs b/src/sqlite_database.rs index fa519ffd0..82bb9d4fc 100644 --- a/src/sqlite_database.rs +++ b/src/sqlite_database.rs @@ -82,7 +82,8 @@ impl Database for SqliteDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - let _ = db_transaction.execute("INSERT OR REPLACE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let _ = db_transaction.execute("INSERT OR IGNORE INTO torrents (info_hash, completed) VALUES (?, ?)", &[info_hash.to_string(), completed.to_string()]); + let _ = db_transaction.execute("UPDATE torrents SET completed = ? WHERE info_hash = ?", &[completed.to_string(), info_hash.to_string()]); } let _ = db_transaction.commit(); From 84c6ca460c0f8d6a6bb1c63e1203bfe1530439a0 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 17:04:52 +0200 Subject: [PATCH 4/8] Speeding up inserting --- src/mysql_database.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index eea2190a3..ac8bbb54a 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -80,12 +80,24 @@ impl Database for MysqlDatabase { let mut db_transaction = conn.start_transaction(TxOpts::default()).map_err(|_| database::Error::DatabaseError)?; + let mut insert_vector= vec![]; + for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX(?), ?) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", (info_hash.to_string(), completed.to_string())).is_err() { + insert_vector.push("(UNHEX('" + info_hash.to_string() + "'), " + completed.to_string() + ")"); + if insert_vector.len() == 1000 { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { + return Err(Error::InvalidQuery); + } + insert_vector.clear(); + } + } + + if insert_vector.len() != 0 { + if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { return Err(Error::InvalidQuery); } - debug!("INSERT INTO torrents (info_hash, completed) VALUES (UNHEX('{}'), {}) ON DUPLICATE KEY UPDATE completed = VALUES(completed)", info_hash.to_string(), completed.to_string()); + insert_vector.clear(); } if db_transaction.commit().is_err() { From 8ebd1e5f1164dff111049efcda15a43bd1277a1b Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 17:13:19 +0200 Subject: [PATCH 5/8] Typo's --- src/mysql_database.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mysql_database.rs b/src/mysql_database.rs index ac8bbb54a..15e2de633 100644 --- a/src/mysql_database.rs +++ b/src/mysql_database.rs @@ -84,9 +84,10 @@ impl Database for MysqlDatabase { for (info_hash, torrent_entry) in torrents { let (_seeders, completed, _leechers) = torrent_entry.get_stats(); - insert_vector.push("(UNHEX('" + info_hash.to_string() + "'), " + completed.to_string() + ")"); + insert_vector.push(format!("(UNHEX('{}'), {})", info_hash.to_string(), completed.to_string())); if insert_vector.len() == 1000 { - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { + let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); + if db_transaction.query_drop(query).is_err() { return Err(Error::InvalidQuery); } insert_vector.clear(); @@ -94,7 +95,8 @@ impl Database for MysqlDatabase { } if insert_vector.len() != 0 { - if db_transaction.exec_drop("INSERT INTO torrents (info_hash, completed) VALUES " + insert_vector.join(",") + "ON DUPLICATE KEY UPDATE completed = VALUES(completed)").is_err() { + let query = format!("INSERT INTO torrents (info_hash, completed) VALUES {} ON DUPLICATE KEY UPDATE completed = VALUES(completed)", insert_vector.join(",")); + if db_transaction.query_drop(query).is_err() { return Err(Error::InvalidQuery); } insert_vector.clear(); From 1b4decac685b374cf1b4abab9cae64fb98a21ebc Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 22:06:33 +0200 Subject: [PATCH 6/8] Looking for reason of timeout --- src/tracker.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tracker.rs b/src/tracker.rs index a67cb62e6..c523cefba 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -308,6 +308,7 @@ impl TorrentTracker { let mut updates = self.updates.write().await; let mut updates_cloned: std::collections::HashMap = std::collections::HashMap::new(); // let mut torrent_hashes: Vec = Vec::new(); + info!("Copying updates to updates_cloned..."); for (k, completed) in updates.iter() { updates_cloned.insert(k.clone(), completed.clone()); } @@ -315,6 +316,7 @@ impl TorrentTracker { drop(updates); let mut shadows = self.shadow.write().await; + info!("Copying updates_cloned into the shadow to overwrite..."); for (k, completed) in updates_cloned.iter() { if shadows.contains_key(k) { shadows.remove(k); @@ -324,6 +326,7 @@ impl TorrentTracker { drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. + info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { @@ -338,11 +341,15 @@ impl TorrentTracker { // We will now save the data from the shadow into the database. // This should not put any strain on the server itself, other then the harddisk/ssd. + info!("Start saving shadow data into SQL..."); let result = self.database.save_persistent_torrent_data(&shadow_copy).await; if result.is_ok() { + info!("Done saving data to SQL and succeeded, emptying shadow..."); let mut shadow = self.shadow.write().await; shadow.clear(); drop(shadow); + } else { + info!("Done saving data to SQL and failed, not emptying shadow..."); } } } From 035c28d9cc6d52c83e08eda87b99b1562db60b75 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 22:10:16 +0200 Subject: [PATCH 7/8] Improving locking --- src/tracker.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tracker.rs b/src/tracker.rs index c523cefba..8ad1faf2f 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -315,19 +315,21 @@ impl TorrentTracker { updates.clear(); drop(updates); - let mut shadows = self.shadow.write().await; info!("Copying updates_cloned into the shadow to overwrite..."); for (k, completed) in updates_cloned.iter() { + let mut shadows = self.shadow.write().await; if shadows.contains_key(k) { shadows.remove(k); } shadows.insert(k.clone(), completed.clone()); + drop(shadows); } drop(updates_cloned); // We updated the shadow data from the updates data, let's handle shadow data as expected. info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); + let mut shadows = self.shadow.write().await; for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(), @@ -335,8 +337,6 @@ impl TorrentTracker { seeders: 0, }); } - - // Drop the lock drop(shadows); // We will now save the data from the shadow into the database. From aea2de04851058a58faf940f20cff65b315605f4 Mon Sep 17 00:00:00 2001 From: Power2All Date: Sun, 17 Apr 2022 22:12:11 +0200 Subject: [PATCH 8/8] Typo --- src/tracker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracker.rs b/src/tracker.rs index 8ad1faf2f..c0c25bc41 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -329,7 +329,7 @@ impl TorrentTracker { // We updated the shadow data from the updates data, let's handle shadow data as expected. info!("Handle shadow_copy to be updated into SQL..."); let mut shadow_copy: BTreeMap = BTreeMap::new(); - let mut shadows = self.shadow.write().await; + let shadows = self.shadow.read().await; for (infohash, completed) in shadows.iter() { shadow_copy.insert(infohash.clone(), TorrentEntry { peers: Default::default(),