replace opentracker with built in js tracker

Author: tdjsnelling

api/package.json

@@ -15,14 +15,14 @@
     "@sentry/tracing": "^7.36.0",
     "bcrypt": "^5.0.1",
     "bencode": "^2.0.1",
+    "bittorrent-tracker": "9.19.0",
     "body-parser": "^1.19.0",
     "chalk": "^4.1.1",
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
     "dotenv": "^10.0.0",
     "express": "^4.17.1",
     "express-rate-limit": "^6.7.0",
-    "http-proxy-middleware": "^2.0.1",
     "jsonwebtoken": "^8.5.1",
     "memoizee": "^0.4.15",
     "mongoose": "^5.13.2",

api/src/controllers/moderation.js

@@ -1,11 +1,11 @@
-import fetch from 'node-fetch'
 import Report from '../schema/report'
 import Torrent from '../schema/torrent'
 import User from '../schema/user'
 import Progress from '../schema/progress'
 import Invite from '../schema/invite'
 import Request from '../schema/request'
 import Comment from '../schema/comment'
+import { all } from 'express/lib/application'
 
 export const createReport = async (req, res) => {
   if (req.body.reason) {
@@ -158,7 +158,7 @@ export const setReportResolved = async (req, res) => {
   }
 }
 
-export const getStats = async (req, res) => {
+export const getStats = (tracker) => async (req, res) => {
   try {
     if (req.userRole !== 'admin') {
       res.status(401).send('You do not have permission to view tracker stats')
@@ -177,7 +177,37 @@ export const getStats = async (req, res) => {
     })
     const totalComments = await Comment.countDocuments()
 
-    const statsData = {
+    const allPeers = {}
+    let activeTorrents = 0
+
+    Object.keys(tracker.torrents).forEach((infoHash) => {
+      const { peers } = tracker.torrents[infoHash]
+      const keys = peers.keys
+      if (keys.length > 0) activeTorrents++
+
+      keys.forEach((peerId) => {
+        // Don't mark the peer as most recently used for stats
+        const peer = peers.peek(peerId)
+        if (peer == null) return // peers.peek() can evict the peer
+
+        if (!allPeers[peerId]) {
+          allPeers[peerId] = {
+            seeder: false,
+            leecher: false,
+          }
+        }
+
+        if (peer.complete) {
+          allPeers[peerId].seeder = true
+        } else {
+          allPeers[peerId].leecher = true
+        }
+
+        allPeers[peerId].peerId = peer.peerId
+      })
+    })
+
+    res.json({
       registeredUsers,
       bannedUsers,
       uploadedTorrents,
@@ -187,38 +217,17 @@ export const getStats = async (req, res) => {
       totalRequests,
       filledRequests,
       totalComments,
-    }
-
-    try {
-      const trackerRes = await fetch(`${process.env.SQ_TRACKER_URL}/stats`)
-
-      if (!trackerRes.ok) {
-        const body = await trackerRes.text()
-        throw new Error(
-          `Error performing tracker scrape: ${trackerRes.status} ${body}`
-        )
-      }
-
-      const body = await trackerRes.text()
-      const [peers, seeds, activeTorrentsLine] = body.split('\n')
-
-      const leechers = parseInt(peers) - parseInt(seeds)
-
-      const activeTorrentsRegex = /opentracker serving ([0-9]+) torrents/
-      const [, activeTorrents] = activeTorrentsLine.match(activeTorrentsRegex)
-
-      res.json({ ...statsData, peers, seeds, leechers, activeTorrents })
-    } catch (e) {
-      console.error('[DEBUG] Error: could not fetch stats from tracker')
-      res.json({
-        ...statsData,
-        peers: '?',
-        seeds: '?',
-        leechers: '?',
-        activeTorrents: '?',
-      })
-    }
+      activeTorrents,
+      peers: Object.keys(allPeers).length,
+      seeders: Object.values(allPeers).filter(
+        (peer) => peer.seeder && !peer.leecher
+      ).length,
+      leechers: Object.values(allPeers).filter(
+        (peer) => peer.leecher && !peer.seeder
+      ).length,
+    })
   } catch (e) {
+    console.error(e)
     res.status(500).send(e.message)
   }
 }

api/src/controllers/rss.js

@@ -26,7 +26,7 @@ const getTorrentXml = (torrent, userId) => {
     </item>`
 }
 
-export const rssFeed = async (req, res) => {
+export const rssFeed = (tracker) => async (req, res) => {
   const { username, password } = req.cookies
   const { query } = req.query
 
@@ -67,6 +67,7 @@ export const rssFeed = async (req, res) => {
     }
 
     const torrentsWithScrape = await embellishTorrentsWithTrackerScrape(
+      tracker,
       torrents
     )
 

api/src/controllers/torrent.js

@@ -1,45 +1,22 @@
 import bencode from 'bencode'
 import crypto from 'crypto'
-import fetch from 'node-fetch'
 import mongoose from 'mongoose'
-import qs from 'qs'
 import slugify from 'slugify'
 import Torrent from '../schema/torrent'
 import User from '../schema/user'
 import Comment from '../schema/comment'
-import { hexToBinary } from '../middleware/announce'
+import { tracker } from '../index'
 
-export const embellishTorrentsWithTrackerScrape = async (torrents) => {
+export const embellishTorrentsWithTrackerScrape = async (tracker, torrents) => {
   if (!torrents.length) return []
 
   try {
-    const infoHashes = torrents.map((torrent) => hexToBinary(torrent.infoHash))
-    const query = qs.stringify(
-      { info_hash: infoHashes },
-      { encoder: escape, indices: false }
-    )
-
-    const trackerRes = await fetch(
-      `${process.env.SQ_TRACKER_URL}/scrape?${query}`
-    )
-
-    if (!trackerRes.ok) {
-      const body = await trackerRes.text()
-      throw new Error(
-        `[DEBUG] Error performing tracker scrape: ${trackerRes.status} ${body}`
-      )
-    }
-
-    const bencoded = await trackerRes.arrayBuffer()
-    const scrape = bencode.decode(bencoded)
-
     return torrents.map((torrent) => {
-      const scrapeForInfoHash =
-        scrape.files[Buffer.from(hexToBinary(torrent.infoHash), 'binary')]
+      const torrentFromTracker = tracker.torrents[torrent.infoHash]
       return {
         ...torrent,
-        seeders: scrapeForInfoHash?.complete || 0,
-        leechers: scrapeForInfoHash?.incomplete || 0,
+        seeders: torrentFromTracker?.complete || 0,
+        leechers: torrentFromTracker?.incomplete || 0,
       }
     })
   } catch (e) {
@@ -169,7 +146,7 @@ export const downloadTorrent = async (req, res) => {
   res.end()
 }
 
-export const fetchTorrent = async (req, res) => {
+export const fetchTorrent = (tracker) => async (req, res) => {
   const { infoHash } = req.params
 
   try {
@@ -264,9 +241,10 @@ export const fetchTorrent = async (req, res) => {
 
     if (torrent.anonymous) delete torrent.uploadedBy
 
-    const [embellishedTorrent] = await embellishTorrentsWithTrackerScrape([
-      torrent,
-    ])
+    const [embellishedTorrent] = await embellishTorrentsWithTrackerScrape(
+      tracker,
+      [torrent]
+    )
 
     res.json(embellishedTorrent)
   } catch (e) {
@@ -309,6 +287,7 @@ export const getTorrentsPage = async ({
   category,
   tag,
   userId,
+  tracker,
 }) => {
   const torrents = await Torrent.aggregate([
     {
@@ -442,31 +421,32 @@ export const getTorrentsPage = async ({
   ])
 
   return {
-    torrents: await embellishTorrentsWithTrackerScrape(torrents),
+    torrents: await embellishTorrentsWithTrackerScrape(tracker, torrents),
     ...count,
   }
 }
 
-export const listLatest = async (req, res) => {
+export const listLatest = (tracker) => async (req, res) => {
   let { count } = req.query
   count = parseInt(count) || 25
   count = Math.min(count, 100)
   try {
-    const { torrents } = await getTorrentsPage({ limit: count })
+    const { torrents } = await getTorrentsPage({ limit: count, tracker })
     res.json(torrents)
   } catch (e) {
     res.status(500).send(e.message)
   }
 }
 
-export const searchTorrents = async (req, res) => {
+export const searchTorrents = (tracker) => async (req, res) => {
   const { query, category, tag, page } = req.query
   try {
     const torrents = await getTorrentsPage({
       skip: page ? parseInt(page) : 0,
       query: decodeURIComponent(query),
       category,
       tag,
+      tracker,
     })
     res.json(torrents)
   } catch (e) {

api/src/controllers/user.js

@@ -9,7 +9,7 @@ import Progress from '../schema/progress'
 import { getTorrentsPage } from './torrent'
 import { getUserRatio } from '../utils/ratio'
 import { mail } from '../index'
-import { BYTES_GB } from '../middleware/announce'
+import { BYTES_GB } from '../tracker/announce'
 
 export const sendVerificationEmail = async (address, token) => {
   await mail.sendMail({
@@ -285,7 +285,7 @@ export const generateInvite = async (req, res) => {
     const createdInvite = await invite.save()
 
     if (createdInvite) {
-      mail.sendMail({
+      await mail.sendMail({
         from: `"${process.env.SQ_SITE_NAME}" <${process.env.SQ_MAIL_FROM_ADDRESS}>`,
         to: email,
         subject: 'Invite',
@@ -451,7 +451,7 @@ export const finalisePasswordReset = async (req, res) => {
   }
 }
 
-export const fetchUser = async (req, res) => {
+export const fetchUser = (tracker) => async (req, res) => {
   try {
     const { username } = req.params
 
@@ -676,7 +676,7 @@ export const fetchUser = async (req, res) => {
     const { ratio } = await getUserRatio(user._id)
     user.ratio = ratio
 
-    const { torrents } = await getTorrentsPage({ userId: user._id })
+    const { torrents } = await getTorrentsPage({ userId: user._id, tracker })
     user.torrents = torrents
 
     res.json(user)

api/src/index.js

@@ -7,16 +7,13 @@ import cors from 'cors'
 import mongoose from 'mongoose'
 import nodemailer from 'nodemailer'
 import ratelimit from 'express-rate-limit'
+import Tracker from 'bittorrent-tracker'
 import * as Sentry from '@sentry/node'
 import * as Tracing from '@sentry/tracing'
 import config from '../../config'
 import validateConfig from './utils/validateConfig'
-import handleAnnounce from './middleware/announce'
+import createTrackerRoute from './tracker/routes'
 import auth from './middleware/auth'
-import {
-  createUserTrackerRoutes,
-  createOtherTrackerRoutes,
-} from './routes/tracker'
 import {
   register,
   login,
@@ -173,15 +170,15 @@ validateConfig(config).then(() => {
   })
   app.use(limiter)
 
-  // custom logic implementing user tracking, ratio control etc
-  app.use('/sq/*/announce', handleAnnounce)
-
-  // proxy and manipulate tracker routes
-  const userTrackerRoutes = createUserTrackerRoutes()
-  const otherTrackerRoutes = createOtherTrackerRoutes()
-  app.use('/sq/*/announce', userTrackerRoutes)
-  app.use('/sq/*/scrape', userTrackerRoutes)
-  app.use('/stats', otherTrackerRoutes)
+  const tracker = new Tracker.Server({
+    http: false,
+    udp: false,
+    ws: false,
+    trustProxy: true,
+  })
+  const onTrackerRequest = tracker._onRequest.bind(tracker)
+  app.get('/sq/*/announce', createTrackerRoute('announce', onTrackerRequest))
+  app.get('/sq/*/scrape', createTrackerRoute('scrape', onTrackerRequest))
 
   app.use(bodyParser.json({ limit: '5mb' }))
   app.use(cookieParser())
@@ -200,7 +197,7 @@ validateConfig(config).then(() => {
   app.post('/verify-email', verifyUserEmail)
 
   // rss feed (auth handled in cookies)
-  app.get('/rss', rssFeed)
+  app.get('/rss', rssFeed(tracker))
 
   // torrent file download (can download without auth, will not be able to announce)
   app.get('/torrent/download/:infoHash/:userId', downloadTorrent)
@@ -216,7 +213,7 @@ validateConfig(config).then(() => {
   app.get('/account/get-role', getUserRole)
   app.get('/account/get-verified', getUserVerifiedEmailStatus)
   app.post('/account/buy', buyItems)
-  app.get('/user/:username', fetchUser)
+  app.get('/user/:username', fetchUser(tracker))
   app.post('/user/ban/:username', banUser)
   app.post('/user/unban/:username', unbanUser)
   app.get('/account/totp/generate', generateTotpSecret)
@@ -225,15 +222,15 @@ validateConfig(config).then(() => {
 
   // torrent routes
   app.post('/torrent/upload', uploadTorrent)
-  app.get('/torrent/info/:infoHash', fetchTorrent)
+  app.get('/torrent/info/:infoHash', fetchTorrent(tracker))
   app.delete('/torrent/delete/:infoHash', deleteTorrent)
   app.post('/torrent/comment/:infoHash', addCommentTorrent)
   app.post('/torrent/vote/:infoHash/:vote', addVote)
   app.post('/torrent/unvote/:infoHash/:vote', removeVote)
   app.post('/torrent/report/:infoHash', createReport)
   app.post('/torrent/toggle-freeleech/:infoHash', toggleFreeleech)
-  app.get('/torrents/latest', listLatest)
-  app.get('/torrents/search', searchTorrents)
+  app.get('/torrents/latest', listLatest(tracker))
+  app.get('/torrents/search', searchTorrents(tracker))
 
   // announcement routes
   app.post('/announcements/new', createAnnouncement)
@@ -249,7 +246,7 @@ validateConfig(config).then(() => {
   app.get('/reports/page/:page', getReports)
   app.post('/reports/resolve/:reportId', setReportResolved)
   app.get('/reports/:reportId', fetchReport)
-  app.get('/admin/stats', getStats)
+  app.get('/admin/stats', getStats(tracker))
 
   // request routes
   app.post('/requests/new', createRequest)