Merge pull request #2 from robertpelloni/feature/megatorrent-reference-2716865330907683697

Feature/megatorrent reference 2716865330907683697

Author: robertpelloni

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "qbittorrent"]
+	path = qbittorrent
+	url = https://github.com/robertpelloni/qbittorrent

ARCHITECTURE.md

@@ -0,0 +1,21 @@
+# Megatorrent Monorepo
+
+This repository contains the reference implementation for the Megatorrent protocol, a decentralized, mutable successor to BitTorrent.
+
+## Structure
+
+*   `docs/`: Architecture and Protocol Specifications.
+*   `tracker/`: Node.js implementation of the Tracker and Reference Client (the root of this repo, historically).
+*   `qbittorrent/`: C++ Client Fork (Submodule).
+
+## Getting Started
+
+### Node.js Tracker & Reference Client
+Run the tracker and client tests:
+```bash
+npm install
+npm test
+```
+
+### qBittorrent Client
+See `qbittorrent/README.md` for C++ build instructions.

README-monorepo.md

@@ -0,0 +1,49 @@
+
+## 6. Obfuscated Storage Protocol (Phase 2)
+To achieve plausible deniability and distributed redundancy, data is not stored as "files" but as "Blobs" of high-entropy data.
+
+### Concepts
+*   **Blob**: A fixed-size (or variable) container stored by a Host. To the Host, it is just random bytes.
+*   **Chunk**: A segment of a user's file.
+*   **Encryption**: Each Chunk is encrypted with a *unique* random key (ChaCha20-Poly1305) before being placed into a Blob.
+*   **Muxing**: A Blob may contain multiple Chunks (potentially from different files) or padding.
+
+### Data Structure: The `FileEntry`
+This structure replaces the simple "magnet link" in the Megatorrent Manifest.
+
+```json
+{
+  "name": "episode1.mkv",
+  "mime": "video/x-matroska",
+  "size": 104857600,
+  "chunks": [
+    {
+      "blobId": "<SHA256 Hash of the Blob>",
+      "offset": 0,         // Byte offset in the Blob
+      "length": 1048576,   // Length of the encrypted chunk
+      "key": "<32-byte Hex Key>",
+      "nonce": "<12-byte Hex Nonce>",
+      "authTag": "<16-byte Hex Tag>"
+    },
+    ...
+  ]
+}
+```
+
+### Process
+1.  **Ingest**:
+    *   File is split into N chunks.
+    *   For each chunk:
+        *   Generate random Key & Nonce.
+        *   Encrypt Chunk -> EncryptedChunk.
+        *   (Simplification for Ref Impl) EncryptedChunk becomes a "Blob" directly (or is wrapped).
+        *   Blob ID = SHA256(Blob).
+    *   Result: A list of Blobs (to be uploaded) and a `FileEntry` (to be put in the Manifest).
+
+2.  **Access**:
+    *   Subscriber receives Manifest.
+    *   Parses `FileEntry`.
+    *   Requests Blob(ID) from the network (simulated via local dir or tracker relay).
+    *   Extracts bytes at `offset` for `length`.
+    *   Decrypts using `key` and `nonce`.
+    *   Reassembles file.

docs/ARCHITECTURE.md

@@ -10,6 +10,17 @@ export function validateManifest (manifest) {
   if (!manifest || typeof manifest !== 'object') throw new Error('Invalid manifest')
   if (!manifest.publicKey || !manifest.signature) throw new Error('Missing keys')
 
+  // Validation
+  if (typeof manifest.publicKey !== 'string' || !/^[0-9a-fA-F]{64}$/.test(manifest.publicKey)) {
+    throw new Error('Invalid public key format')
+  }
+  if (typeof manifest.signature !== 'string' || !/^[0-9a-fA-F]{128}$/.test(manifest.signature)) {
+    throw new Error('Invalid signature format')
+  }
+  if (typeof manifest.sequence !== 'number') throw new Error('Invalid sequence')
+  if (typeof manifest.timestamp !== 'number') throw new Error('Invalid timestamp')
+  if (!Array.isArray(manifest.collections)) throw new Error('Invalid collections')
+
   // Reconstruct the payload to verify (exclude signature)
   const payload = {
     publicKey: manifest.publicKey,

lib/manifest.js

@@ -6,17 +6,20 @@ import minimist from 'minimist'
 import Client from 'bittorrent-tracker'
 import { generateKeypair } from './lib/crypto.js'
 import { createManifest, validateManifest } from './lib/manifest.js'
+import { ingest, reassemble } from './lib/storage.js'
 
 const argv = minimist(process.argv.slice(2), {
   alias: {
     k: 'keyfile',
     t: 'tracker',
     i: 'input',
-    o: 'output'
+    o: 'output',
+    d: 'dir'
   },
   default: {
     keyfile: './identity.json',
-    tracker: 'ws://localhost:8000' // Default to local WS tracker
+    tracker: 'ws://localhost:8000', // Default to local WS tracker
+    dir: './storage'
   }
 })
 
@@ -25,12 +28,18 @@ const command = argv._[0]
 if (!command) {
   console.error(`Usage:
   gen-key [-k identity.json]
-  publish [-k identity.json] [-t ws://tracker] -i magnet_list.txt
-  subscribe [-t ws://tracker] <public_key_hex>
+  ingest -i <file> [-d ./storage] -> Returns FileEntry JSON
+  publish [-k identity.json] [-t ws://tracker] -i <file_entry.json>
+  subscribe [-t ws://tracker] <public_key_hex> [-d ./storage]
   `)
   process.exit(1)
 }
 
+// Ensure storage dir exists
+if (!fs.existsSync(argv.dir)) {
+  fs.mkdirSync(argv.dir, { recursive: true })
+}
+
 // 1. Generate Key
 if (command === 'gen-key') {
   const keypair = generateKeypair()
@@ -44,7 +53,27 @@ if (command === 'gen-key') {
   process.exit(0)
 }
 
-// 2. Publish
+// 2. Ingest
+if (command === 'ingest') {
+  if (!argv.input) {
+    console.error('Please specify input file with -i')
+    process.exit(1)
+  }
+  const fileBuf = fs.readFileSync(argv.input)
+  const result = ingest(fileBuf, path.basename(argv.input))
+
+  // Save Blobs
+  result.blobs.forEach(blob => {
+    fs.writeFileSync(path.join(argv.dir, blob.id), blob.buffer)
+  })
+
+  console.log(`Ingested ${result.blobs.length} blobs to ${argv.dir}`)
+  console.log('FileEntry JSON (save this to a file to publish it):')
+  console.log(JSON.stringify(result.fileEntry, null, 2))
+  process.exit(0)
+}
+
+// 3. Publish
 if (command === 'publish') {
   if (!fs.existsSync(argv.keyfile)) {
     console.error('Keyfile not found. Run gen-key first.')
@@ -56,23 +85,32 @@ if (command === 'publish') {
     secretKey: Buffer.from(keyData.secretKey, 'hex')
   }
 
-  // Read magnet links
+  // Read Input
   if (!argv.input) {
-    console.error('Please specify input file with -i (list of magnet links)')
+    console.error('Please specify input file with -i (json file entry or text list)')
     process.exit(1)
   }
+
   const content = fs.readFileSync(argv.input, 'utf-8')
-  const lines = content.split('\n').map(l => l.trim()).filter(l => l.length > 0)
+  let items
+  try {
+    // Try parsing as JSON (FileEntry)
+    const json = JSON.parse(content)
+    // Wrap in our "Items" list.
+    // In the future, "Items" can be Magnet Links OR FileEntries.
+    items = [json]
+  } catch (e) {
+    // Fallback: Line-separated magnet links
+    items = content.split('\n').map(l => l.trim()).filter(l => l.length > 0)
+  }
 
-  // Create Collections structure (flat for now)
+  // Create Collections structure
   const collections = [{
     title: 'Default Collection',
-    items: lines
+    items
   }]
 
   // Create Manifest
-  // TODO: Retrieve last sequence from somewhere or store it?
-  // For now, we use timestamp as sequence to be simple and monotonic
   const sequence = Date.now()
   const manifest = createManifest(keypair, sequence, collections)
 
@@ -123,7 +161,7 @@ if (command === 'publish') {
   }, 1000)
 }
 
-// 3. Subscribe
+// 4. Subscribe
 if (command === 'subscribe') {
   const pubKeyHex = argv._[1]
   if (!pubKeyHex) {
@@ -166,7 +204,37 @@ if (command === 'subscribe') {
             if (valid && data.manifest.publicKey === pubKeyHex) {
               console.log('VERIFIED UPDATE from ' + pubKeyHex)
               console.log('Sequence:', data.manifest.sequence)
-              console.log('Items:', data.manifest.collections[0].items)
+
+              const items = data.manifest.collections[0].items
+              console.log(`Received ${items.length} items.`)
+
+              // Auto-Download Logic (Prototype)
+              items.forEach(async (item, idx) => {
+                if (typeof item === 'object' && item.chunks) {
+                  console.log(`Item ${idx}: Detected Megatorrent FileEntry: ${item.name}`)
+                  try {
+                    const fileBuf = await reassemble(item, async (blobId) => {
+                      const p = path.join(argv.dir, blobId)
+                      if (fs.existsSync(p)) {
+                        return fs.readFileSync(p)
+                      }
+                      // TODO: Network fetch
+                      console.log(`Blob ${blobId} not found locally.`)
+                      return null
+                    })
+
+                    if (fileBuf) {
+                      const outPath = path.join(argv.dir, 'downloaded_' + item.name)
+                      fs.writeFileSync(outPath, fileBuf)
+                      console.log(`SUCCESS: Reassembled to ${outPath}`)
+                    }
+                  } catch (err) {
+                    console.error('Failed to reassemble:', err.message)
+                  }
+                } else {
+                  console.log(`Item ${idx}: Standard Magnet/Text: ${item}`)
+                }
+              })
             } else {
               console.error('Invalid signature or wrong key!')
             }