Implement Reference Successor Protocol (Megatorrent)

ARCHITECTURE.md

@@ -0,0 +1,79 @@
+# Reference Successor Protocol Architecture
+
+## Overview
+This document outlines the reference implementation for a successor to the BitTorrent protocol, focusing on:
+1.  **Mutable Subscriptions ("Megatorrents")**: Authenticated, updateable lists of content.
+2.  **Identity**: Strong cryptographic ownership (Ed25519) without central authority.
+3.  **Decentralized Tracking**: A hybrid approach where trackers facilitate discovery and update propagation, but are not the sole source of truth.
+4.  **Future: Distributed Encrypted Storage**: A vision for obfuscated, sharded, and encrypted data storage (see Future Work).
+
+## 1. Identity
+*   **Algorithm**: Ed25519 (Edwards-curve Digital Signature Algorithm).
+*   **Keypair**:
+    *   `PublicKey` (32 bytes): Acts as the immutable "Channel ID" or "Address".
+    *   `SecretKey` (64 bytes): Held by the author to sign updates.
+*   **Addressing**: A "subscription" is simply the Hex-encoded Public Key.
+
+## 2. Megatorrent Manifest
+A "Megatorrent" is a dynamic collection of standard torrents (or future encrypted chunks).
+
+### Format (JSON)
+```json
+{
+  "publicKey": "<Hex encoded Ed25519 Public Key>",
+  "sequence": 1, // Monotonically increasing integer
+  "timestamp": 1678886400000,
+  "collections": [
+    {
+      "title": "Season 1",
+      "items": [
+        "magnet:?xt=urn:btih:hash1&dn=ep1...",
+        "magnet:?xt=urn:btih:hash2&dn=ep2..."
+      ]
+    }
+  ],
+  "signature": "<Hex encoded Ed25519 Signature of the canonical JSON>"
+}
+```
+
+### Validation Rules
+1.  `signature` must be valid for `publicKey` over the content (excluding the signature field itself).
+2.  `sequence` must be greater than the previously known sequence for this key (replay protection).
+
+## 3. Tracker Protocol Extensions
+The standard BitTorrent tracker protocol (HTTP/WS) is extended with two new actions.
+
+### Action: `publish` (WebSocket)
+*   **Direction**: Client -> Tracker
+*   **Params**:
+    *   `action`: "publish"
+    *   `manifest`: (The JSON object above)
+*   **Tracker Logic**:
+    *   Validate signature.
+    *   Check `sequence` > stored sequence.
+    *   Store/Cache manifest.
+    *   Broadcast to all active subscribers.
+
+### Action: `subscribe` (WebSocket)
+*   **Direction**: Client -> Tracker
+*   **Params**:
+    *   `action`: "subscribe"
+    *   `key`: "<Hex Public Key>"
+*   **Tracker Logic**:
+    *   Add socket to subscription list for `key`.
+    *   Immediately send the latest cached `publish` message (if any).
+
+## 4. Client Implementation ("Reference Client")
+The reference client demonstrates:
+1.  **Key Generation**: Creating a permanent identity.
+2.  **Publishing**: Reading a list of magnet links and pushing an update.
+3.  **Subscribing**: Watching a key and receiving real-time updates.
+4.  **Anonymity**: SOCKS5 support for running over Tor/I2P.
+
+## 5. Future Work: Distributed Encrypted Storage
+The ultimate goal includes a storage layer with:
+*   **Muxed Encrypted Chunks**: Files split into chunks, encrypted, and mixed with others.
+*   **Plausible Deniability**: Host nodes store opaque blobs without knowing content.
+*   **Dynamic Reassembly**: Files reassembled via a manifest of chunk hashes, potentially with random offsets/padding.
+
+*Note: This reference implementation focuses on the Control Plane (Subscriptions) first.*

lib/manifest.js

@@ -0,0 +1,26 @@
+import stringify from 'fast-json-stable-stringify'
+import sodium from 'sodium-native'
+
+export function verify (message, signature, publicKey) {
+  const msgBuffer = Buffer.isBuffer(message) ? message : Buffer.from(message)
+  return sodium.crypto_sign_verify_detached(signature, msgBuffer, publicKey)
+}
+
+export function validateManifest (manifest) {
+  if (!manifest || typeof manifest !== 'object') throw new Error('Invalid manifest')
+  if (!manifest.publicKey || !manifest.signature) throw new Error('Missing keys')
+
+  // Reconstruct the payload to verify (exclude signature)
+  const payload = {
+    publicKey: manifest.publicKey,
+    sequence: manifest.sequence,
+    timestamp: manifest.timestamp,
+    collections: manifest.collections
+  }
+
+  const jsonString = stringify(payload)
+  const publicKey = Buffer.from(manifest.publicKey, 'hex')
+  const signature = Buffer.from(manifest.signature, 'hex')
+
+  return verify(jsonString, signature, publicKey)
+}

lib/server/parse-websocket.js

@@ -48,6 +48,12 @@ export default function (socket, opts, params) {
         return bin2hex(binaryInfoHash)
       })
     }
+  } else if (params.action === 'publish') {
+    // Custom Action: Publish Manifest
+    if (!params.manifest) throw new Error('missing manifest')
+  } else if (params.action === 'subscribe') {
+    // Custom Action: Subscribe to Key
+    if (!params.key) throw new Error('missing key')
   } else {
     throw new Error(`invalid action in WS request: ${params.action}`)
   }

package.json

@@ -35,6 +35,7 @@
     "compact2string": "^1.4.1",
     "cross-fetch-ponyfill": "^1.0.3",
     "debug": "^4.3.4",
+    "fast-json-stable-stringify": "^2.1.0",
     "ip": "^2.0.1",
     "lru": "^3.1.0",
     "minimist": "^1.2.8",
@@ -44,6 +45,7 @@
     "run-parallel": "^1.2.0",
     "run-series": "^1.1.9",
     "socks": "^2.8.3",
+    "sodium-native": "^5.0.10",
     "string2compact": "^2.0.1",
     "uint8-util": "^2.2.5",
     "unordered-array-remove": "^1.0.2",
@@ -52,7 +54,7 @@
   "devDependencies": {
     "@mapbox/node-pre-gyp": "1.0.11",
     "@webtorrent/semantic-release-config": "1.0.10",
-    "magnet-uri": "7.0.7",
+    "magnet-uri": "^7.0.7",
     "semantic-release": "21.1.2",
     "standard": "*",
     "tape": "5.9.0",

reference-client/index.js

@@ -0,0 +1,180 @@
+#!/usr/bin/env node
+
+import fs from 'fs'
+import path from 'path'
+import minimist from 'minimist'
+import Client from 'bittorrent-tracker'
+import { generateKeypair } from './lib/crypto.js'
+import { createManifest, validateManifest } from './lib/manifest.js'
+
+const argv = minimist(process.argv.slice(2), {
+  alias: {
+    k: 'keyfile',
+    t: 'tracker',
+    i: 'input',
+    o: 'output'
+  },
+  default: {
+    keyfile: './identity.json',
+    tracker: 'ws://localhost:8000' // Default to local WS tracker
+  }
+})
+
+const command = argv._[0]
+
+if (!command) {
+  console.error(`Usage:
+  gen-key [-k identity.json]
+  publish [-k identity.json] [-t ws://tracker] -i magnet_list.txt
+  subscribe [-t ws://tracker] <public_key_hex>
+  `)
+  process.exit(1)
+}
+
+// 1. Generate Key
+if (command === 'gen-key') {
+  const keypair = generateKeypair()
+  const data = {
+    publicKey: keypair.publicKey.toString('hex'),
+    secretKey: keypair.secretKey.toString('hex')
+  }
+  fs.writeFileSync(argv.keyfile, JSON.stringify(data, null, 2))
+  console.log(`Identity generated at ${argv.keyfile}`)
+  console.log(`Public Key: ${data.publicKey}`)
+  process.exit(0)
+}
+
+// 2. Publish
+if (command === 'publish') {
+  if (!fs.existsSync(argv.keyfile)) {
+    console.error('Keyfile not found. Run gen-key first.')
+    process.exit(1)
+  }
+  const keyData = JSON.parse(fs.readFileSync(argv.keyfile))
+  const keypair = {
+    publicKey: Buffer.from(keyData.publicKey, 'hex'),
+    secretKey: Buffer.from(keyData.secretKey, 'hex')
+  }
+
+  // Read magnet links
+  if (!argv.input) {
+    console.error('Please specify input file with -i (list of magnet links)')
+    process.exit(1)
+  }
+  const content = fs.readFileSync(argv.input, 'utf-8')
+  const lines = content.split('\n').map(l => l.trim()).filter(l => l.length > 0)
+
+  // Create Collections structure (flat for now)
+  const collections = [{
+    title: 'Default Collection',
+    items: lines
+  }]
+
+  // Create Manifest
+  // TODO: Retrieve last sequence from somewhere or store it?
+  // For now, we use timestamp as sequence to be simple and monotonic
+  const sequence = Date.now()
+  const manifest = createManifest(keypair, sequence, collections)
+
+  console.log('Publishing manifest:', JSON.stringify(manifest, null, 2))
+
+  // Connect to Tracker
+  const client = new Client({
+    infoHash: Buffer.alloc(20), // Dummy, not used for custom proto
+    peerId: Buffer.alloc(20), // Dummy
+    announce: [argv.tracker],
+    port: 6666
+  })
+
+  client.on('error', err => console.error('Client Error:', err.message))
+
+  client.on('update', () => {
+    // We don't expect standard updates
+  })
+
+  // We need to access the underlying socket to send our custom message
+  // bittorrent-tracker abstracts this, so we might need to hook into the `announce` phase or just use the socket directly if exposed.
+  // The library exposes `client._trackers` which is a list of Tracker instances.
+
+  // Wait for socket connection
+  setTimeout(() => {
+    const trackers = client._trackers
+    let sent = false
+
+    for (const tracker of trackers) {
+      // We only support WebSocket for this custom protocol right now
+      if (tracker.socket && tracker.socket.readyState === 1) { // OPEN
+        console.log('Sending publish message to ' + tracker.announceUrl)
+        tracker.socket.send(JSON.stringify({
+          action: 'publish',
+          manifest
+        }))
+        sent = true
+      }
+    }
+
+    if (sent) console.log('Publish sent!')
+    else console.error('No connected websocket trackers found.')
+
+    setTimeout(() => {
+      client.destroy()
+      process.exit(0)
+    }, 1000)
+  }, 1000)
+}
+
+// 3. Subscribe
+if (command === 'subscribe') {
+  const pubKeyHex = argv._[1]
+  if (!pubKeyHex) {
+    console.error('Please provide public key hex')
+    process.exit(1)
+  }
+
+  console.log(`Subscribing to ${pubKeyHex}...`)
+
+  const client = new Client({
+    infoHash: Buffer.alloc(20),
+    peerId: Buffer.alloc(20),
+    announce: [argv.tracker],
+    port: 6667
+  })
+
+  client.on('error', err => console.error('Client Error:', err.message))
+
+  // Hook into internal trackers to send subscribe
+  setInterval(() => {
+    const trackers = client._trackers
+    for (const tracker of trackers) {
+      if (tracker.socket && tracker.socket.readyState === 1 && !tracker._subscribed) {
+        console.log('Sending subscribe to ' + tracker.announceUrl)
+        tracker.socket.send(JSON.stringify({
+          action: 'subscribe',
+          key: pubKeyHex
+        }))
+        tracker._subscribed = true // simple flag to avoid spamming
+
+        // Listen for responses
+        const originalOnMessage = tracker.socket.onmessage
+        tracker.socket.onmessage = (event) => {
+          let data
+          try { data = JSON.parse(event.data) } catch (e) { return }
+
+          if (data.action === 'publish') {
+            console.log('\n>>> RECEIVED UPDATE <<<')
+            const valid = validateManifest(data.manifest)
+            if (valid && data.manifest.publicKey === pubKeyHex) {
+              console.log('VERIFIED UPDATE from ' + pubKeyHex)
+              console.log('Sequence:', data.manifest.sequence)
+              console.log('Items:', data.manifest.collections[0].items)
+            } else {
+              console.error('Invalid signature or wrong key!')
+            }
+          } else {
+            if (originalOnMessage) originalOnMessage(event)
+          }
+        }
+      }
+    }
+  }, 1000)
+}

reference-client/lib/crypto.js

@@ -0,0 +1,20 @@
+import sodium from 'sodium-native'
+
+export function generateKeypair () {
+  const publicKey = Buffer.alloc(sodium.crypto_sign_PUBLICKEYBYTES)
+  const secretKey = Buffer.alloc(sodium.crypto_sign_SECRETKEYBYTES)
+  sodium.crypto_sign_keypair(publicKey, secretKey)
+  return { publicKey, secretKey }
+}
+
+export function sign (message, secretKey) {
+  const signature = Buffer.alloc(sodium.crypto_sign_BYTES)
+  const msgBuffer = Buffer.isBuffer(message) ? message : Buffer.from(message)
+  sodium.crypto_sign_detached(signature, msgBuffer, secretKey)
+  return signature
+}
+
+export function verify (message, signature, publicKey) {
+  const msgBuffer = Buffer.isBuffer(message) ? message : Buffer.from(message)
+  return sodium.crypto_sign_verify_detached(signature, msgBuffer, publicKey)
+}

reference-client/lib/manifest.js

@@ -0,0 +1,41 @@
+import stringify from 'fast-json-stable-stringify'
+import { sign, verify } from './crypto.js'
+
+export function createManifest (keypair, sequence, collections) {
+  const payload = {
+    publicKey: keypair.publicKey.toString('hex'),
+    sequence,
+    timestamp: Date.now(),
+    collections
+  }
+
+  // Canonicalize string for signing
+  const jsonString = stringify(payload)
+
+  // Sign
+  const signature = sign(jsonString, keypair.secretKey)
+
+  return {
+    ...payload,
+    signature: signature.toString('hex')
+  }
+}
+
+export function validateManifest (manifest) {
+  if (!manifest || typeof manifest !== 'object') throw new Error('Invalid manifest')
+  if (!manifest.publicKey || !manifest.signature) throw new Error('Missing keys')
+
+  // Reconstruct the payload to verify (exclude signature)
+  const payload = {
+    publicKey: manifest.publicKey,
+    sequence: manifest.sequence,
+    timestamp: manifest.timestamp,
+    collections: manifest.collections
+  }
+
+  const jsonString = stringify(payload)
+  const publicKey = Buffer.from(manifest.publicKey, 'hex')
+  const signature = Buffer.from(manifest.signature, 'hex')
+
+  return verify(jsonString, signature, publicKey)
+}