implement bittorrent tracker in API service, remove opentracker

README.md

@@ -35,19 +35,15 @@ It implements all of the features required to run a private (or public) tracker
 
 ### Components
 
-An sqtracker deployment is made up of 5 separate components. These are:
-
-#### A BitTorrent tracker
-
-sqtracker does not implement the BitTorrent tracker spec itself. Instead, it works alongside a tracker server such as [opentracker](https://erdgeist.org/arts/software/opentracker/). In theory, other generic BitTorrent tracker software should work, but opentracker is recommended for the time being.
+An sqtracker deployment is made up of 4 separate components. These are:
 
 #### A MongoDB database
 
 [MongoDB](https://www.mongodb.com/) is a popular and powerful document-oriented database. Version 5.2 or higher is required.
 
 #### The sqtracker API service
 
-The sqtracker API service handles all actions taken by users (authentication, uploads, searching etc.), provides the RSS feed, and proxies announce requests to the tracker server. 
+The sqtracker API service handles all actions taken by users (authentication, uploads, searching etc.), implements the BitTorrent tracker specification to handle announces and scrapes, and provides the RSS feed. 
 
 #### The sqtracker client service
 
@@ -88,7 +84,6 @@ If your configuration is not valid, sqtracker will fail to start.
 | SQ_TORRENT_CATEGORIES      | envs    | `["Movies", "TV"]`             | An array of categories available on your tracker site                                                                                                                                                                                                                                        |
 | SQ_BASE_URL                | envs    | https://demo.sqtracker.dev     | The URL of your tracker site                                                                                                                                                                                                                                                                 |
 | SQ_API_URL                 | envs    | https://demo.sqtracker.dev/api | The URL of your API. Under the recommended setup, it should be `${SQ_BASE_URL}/api`                                                                                                                                                                                                          |
-| SQ_TRACKER_URL             | envs    | http://sq_opentracker:6969     | The URL of your tracker server. Under the recommended setup, it should be `http://sq_opentracker:6969`                                                                                                                                                                                       |
 | SQ_MONGO_URL               | envs    | mongodb://sq_mongodb/sq        | The URL of your MongoDB server. Under the recommended setup, it should be `mongodb://sq_mongodb/sq`                                                                                                                                                                                          |
 | SQ_MAIL_FROM_ADDRESS       | envs    | mail@sqtracker.dev             | The address that mail will be sent from                                                                                                                                                                                                                                                      |
 | SQ_SMTP_HOST               | envs    | smtp.example.com               | The hostname of your SMTP server                                                                                                                                                                                                                                                             |

api/package.json

@@ -15,14 +15,14 @@
     "@sentry/tracing": "^7.36.0",
     "bcrypt": "^5.0.1",
     "bencode": "^2.0.1",
+    "bittorrent-tracker": "9.19.0",
     "body-parser": "^1.19.0",
     "chalk": "^4.1.1",
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
     "dotenv": "^10.0.0",
     "express": "^4.17.1",
     "express-rate-limit": "^6.7.0",
-    "http-proxy-middleware": "^2.0.1",
     "jsonwebtoken": "^8.5.1",
     "memoizee": "^0.4.15",
     "mongoose": "^5.13.2",

api/src/controllers/moderation.js

@@ -1,11 +1,11 @@
-import fetch from 'node-fetch'
 import Report from '../schema/report'
 import Torrent from '../schema/torrent'
 import User from '../schema/user'
 import Progress from '../schema/progress'
 import Invite from '../schema/invite'
 import Request from '../schema/request'
 import Comment from '../schema/comment'
+import { all } from 'express/lib/application'
 
 export const createReport = async (req, res) => {
   if (req.body.reason) {
@@ -158,7 +158,7 @@ export const setReportResolved = async (req, res) => {
   }
 }
 
-export const getStats = async (req, res) => {
+export const getStats = (tracker) => async (req, res) => {
   try {
     if (req.userRole !== 'admin') {
       res.status(401).send('You do not have permission to view tracker stats')
@@ -177,7 +177,37 @@ export const getStats = async (req, res) => {
     })
     const totalComments = await Comment.countDocuments()
 
-    const statsData = {
+    const allPeers = {}
+    let activeTorrents = 0
+
+    Object.keys(tracker.torrents).forEach((infoHash) => {
+      const { peers } = tracker.torrents[infoHash]
+      const keys = peers.keys
+      if (keys.length > 0) activeTorrents++
+
+      keys.forEach((peerId) => {
+        // Don't mark the peer as most recently used for stats
+        const peer = peers.peek(peerId)
+        if (peer == null) return // peers.peek() can evict the peer
+
+        if (!allPeers[peerId]) {
+          allPeers[peerId] = {
+            seeder: false,
+            leecher: false,
+          }
+        }
+
+        if (peer.complete) {
+          allPeers[peerId].seeder = true
+        } else {
+          allPeers[peerId].leecher = true
+        }
+
+        allPeers[peerId].peerId = peer.peerId
+      })
+    })
+
+    res.json({
       registeredUsers,
       bannedUsers,
       uploadedTorrents,
@@ -187,38 +217,17 @@ export const getStats = async (req, res) => {
       totalRequests,
       filledRequests,
       totalComments,
-    }
-
-    try {
-      const trackerRes = await fetch(`${process.env.SQ_TRACKER_URL}/stats`)
-
-      if (!trackerRes.ok) {
-        const body = await trackerRes.text()
-        throw new Error(
-          `Error performing tracker scrape: ${trackerRes.status} ${body}`
-        )
-      }
-
-      const body = await trackerRes.text()
-      const [peers, seeds, activeTorrentsLine] = body.split('\n')
-
-      const leechers = parseInt(peers) - parseInt(seeds)
-
-      const activeTorrentsRegex = /opentracker serving ([0-9]+) torrents/
-      const [, activeTorrents] = activeTorrentsLine.match(activeTorrentsRegex)
-
-      res.json({ ...statsData, peers, seeds, leechers, activeTorrents })
-    } catch (e) {
-      console.error('[DEBUG] Error: could not fetch stats from tracker')
-      res.json({
-        ...statsData,
-        peers: '?',
-        seeds: '?',
-        leechers: '?',
-        activeTorrents: '?',
-      })
-    }
+      activeTorrents,
+      peers: Object.keys(allPeers).length,
+      seeders: Object.values(allPeers).filter(
+        (peer) => peer.seeder && !peer.leecher
+      ).length,
+      leechers: Object.values(allPeers).filter(
+        (peer) => peer.leecher && !peer.seeder
+      ).length,
+    })
   } catch (e) {
+    console.error(e)
     res.status(500).send(e.message)
   }
 }

api/src/controllers/rss.js

@@ -26,7 +26,7 @@ const getTorrentXml = (torrent, userId) => {
     </item>`
 }
 
-export const rssFeed = async (req, res) => {
+export const rssFeed = (tracker) => async (req, res) => {
   const { username, password } = req.cookies
   const { query } = req.query
 
@@ -67,6 +67,7 @@ export const rssFeed = async (req, res) => {
     }
 
     const torrentsWithScrape = await embellishTorrentsWithTrackerScrape(
+      tracker,
       torrents
     )
 

api/src/controllers/torrent.js

@@ -1,45 +1,22 @@
 import bencode from 'bencode'
 import crypto from 'crypto'
-import fetch from 'node-fetch'
 import mongoose from 'mongoose'
-import qs from 'qs'
 import slugify from 'slugify'
 import Torrent from '../schema/torrent'
 import User from '../schema/user'
 import Comment from '../schema/comment'
-import { hexToBinary } from '../middleware/announce'
+import { tracker } from '../index'
 
-export const embellishTorrentsWithTrackerScrape = async (torrents) => {
+export const embellishTorrentsWithTrackerScrape = async (tracker, torrents) => {
   if (!torrents.length) return []
 
   try {
-    const infoHashes = torrents.map((torrent) => hexToBinary(torrent.infoHash))
-    const query = qs.stringify(
-      { info_hash: infoHashes },
-      { encoder: escape, indices: false }
-    )
-
-    const trackerRes = await fetch(
-      `${process.env.SQ_TRACKER_URL}/scrape?${query}`
-    )
-
-    if (!trackerRes.ok) {
-      const body = await trackerRes.text()
-      throw new Error(
-        `[DEBUG] Error performing tracker scrape: ${trackerRes.status} ${body}`
-      )
-    }
-
-    const bencoded = await trackerRes.arrayBuffer()
-    const scrape = bencode.decode(bencoded)
-
     return torrents.map((torrent) => {
-      const scrapeForInfoHash =
-        scrape.files[Buffer.from(hexToBinary(torrent.infoHash), 'binary')]
+      const torrentFromTracker = tracker.torrents[torrent.infoHash]
       return {
         ...torrent,
-        seeders: scrapeForInfoHash?.complete || 0,
-        leechers: scrapeForInfoHash?.incomplete || 0,
+        seeders: torrentFromTracker?.complete || 0,
+        leechers: torrentFromTracker?.incomplete || 0,
       }
     })
   } catch (e) {
@@ -169,7 +146,7 @@ export const downloadTorrent = async (req, res) => {
   res.end()
 }
 
-export const fetchTorrent = async (req, res) => {
+export const fetchTorrent = (tracker) => async (req, res) => {
   const { infoHash } = req.params
 
   try {
@@ -264,9 +241,10 @@ export const fetchTorrent = async (req, res) => {
 
     if (torrent.anonymous) delete torrent.uploadedBy
 
-    const [embellishedTorrent] = await embellishTorrentsWithTrackerScrape([
-      torrent,
-    ])
+    const [embellishedTorrent] = await embellishTorrentsWithTrackerScrape(
+      tracker,
+      [torrent]
+    )
 
     res.json(embellishedTorrent)
   } catch (e) {
@@ -309,6 +287,7 @@ export const getTorrentsPage = async ({
   category,
   tag,
   userId,
+  tracker,
 }) => {
   const torrents = await Torrent.aggregate([
     {
@@ -442,31 +421,32 @@ export const getTorrentsPage = async ({
   ])
 
   return {
-    torrents: await embellishTorrentsWithTrackerScrape(torrents),
+    torrents: await embellishTorrentsWithTrackerScrape(tracker, torrents),
     ...count,
   }
 }
 
-export const listLatest = async (req, res) => {
+export const listLatest = (tracker) => async (req, res) => {
   let { count } = req.query
   count = parseInt(count) || 25
   count = Math.min(count, 100)
   try {
-    const { torrents } = await getTorrentsPage({ limit: count })
+    const { torrents } = await getTorrentsPage({ limit: count, tracker })
     res.json(torrents)
   } catch (e) {
     res.status(500).send(e.message)
   }
 }
 
-export const searchTorrents = async (req, res) => {
+export const searchTorrents = (tracker) => async (req, res) => {
   const { query, category, tag, page } = req.query
   try {
     const torrents = await getTorrentsPage({
       skip: page ? parseInt(page) : 0,
       query: decodeURIComponent(query),
       category,
       tag,
+      tracker,
     })
     res.json(torrents)
   } catch (e) {

api/src/controllers/user.js

@@ -9,7 +9,7 @@ import Progress from '../schema/progress'
 import { getTorrentsPage } from './torrent'
 import { getUserRatio } from '../utils/ratio'
 import { mail } from '../index'
-import { BYTES_GB } from '../middleware/announce'
+import { BYTES_GB } from '../tracker/announce'
 
 export const sendVerificationEmail = async (address, token) => {
   await mail.sendMail({
@@ -285,7 +285,7 @@ export const generateInvite = async (req, res) => {
     const createdInvite = await invite.save()
 
     if (createdInvite) {
-      mail.sendMail({
+      await mail.sendMail({
         from: `"${process.env.SQ_SITE_NAME}" <${process.env.SQ_MAIL_FROM_ADDRESS}>`,
         to: email,
         subject: 'Invite',
@@ -451,7 +451,7 @@ export const finalisePasswordReset = async (req, res) => {
   }
 }
 
-export const fetchUser = async (req, res) => {
+export const fetchUser = (tracker) => async (req, res) => {
   try {
     const { username } = req.params
 
@@ -676,7 +676,7 @@ export const fetchUser = async (req, res) => {
     const { ratio } = await getUserRatio(user._id)
     user.ratio = ratio
 
-    const { torrents } = await getTorrentsPage({ userId: user._id })
+    const { torrents } = await getTorrentsPage({ userId: user._id, tracker })
     user.torrents = torrents
 
     res.json(user)