diff --git a/.gitignore b/.gitignore index fe447ef..b7fb5b4 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ typings/ # Development Environment .vscode + +MOT17Labels.zip +benchmark/MOT17/ diff --git a/benchmark.sh b/benchmark.sh new file mode 100755 index 0000000..2ac3873 --- /dev/null +++ b/benchmark.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +if [ ! -d "benchmark" ]; then + wget -nc https://motchallenge.net/data/MOT17Labels.zip + unzip MOT17Labels.zip "train/*" -d "benchmark" && mv "benchmark/train" "benchmark/MOT17" +fi + +for d in benchmark/MOT17/*/ ; do + node main.js --mode motchallenge --input "${d}det/det.txt" && \ + mv "${d}/det/outputTrackerMOT.txt" "benchmark/MOT17/$(basename ${d}).txt" +done + +pushd benchmark/MOT17 +python3 -m motmetrics.apps.eval_motchallenge . . +popd diff --git a/documentation/BENCHMARK.md b/documentation/BENCHMARK.md index 485d227..1f02047 100644 --- a/documentation/BENCHMARK.md +++ b/documentation/BENCHMARK.md @@ -62,7 +62,43 @@ python3 -m motmetrics.apps.eval_motchallenge . . - Result ``` - IDF1 IDP IDR Rcll Prcn GT MT PT ML FP FN IDs FM MOTA MOTP -MOT17-04-DPM 28.6% 34.4% 24.5% 42.8% 60.0% 83 8 43 32 13558 27210 355 549 13.5% 0.224 -OVERALL 28.6% 34.4% 24.5% 42.8% 60.0% 83 8 43 32 13558 27210 355 549 13.5% 0.224 + IDF1 IDP IDR Rcll Prcn GT MT PT ML FP FN IDs FM MOTA MOTP IDt IDa IDm +MOT17-04-DPM 33.5% 53.8% 24.3% 39.0% 86.4% 83 4 45 34 2924 29004 239 393 32.4% 0.217 106 141 10 +OVERALL 33.5% 53.8% 24.3% 39.0% 86.4% 83 4 45 34 2924 29004 239 393 32.4% 0.217 106 141 10 +``` + +### Run full benchmark + +In order to benchmark the tracker on all MOT17 training sequences, delete the directory `benchmark` and run + +```bash +./benchmark.sh +``` + +On the first run, the script will download the full MOT17 training benchmark and unpack it. Then, it runs the tracker on each sequence and outputs the evaluation metrics. + +``` + IDF1 IDP IDR Rcll Prcn GT MT PT ML FP FN IDs FM MOTA MOTP IDt IDa IDm +MOT17-04-SDP 62.2% 74.8% 53.2% 69.0% 97.1% 83 32 39 12 982 14737 121 367 66.7% 0.152 82 46 12 +MOT17-04-DPM 33.5% 53.8% 24.3% 39.0% 86.4% 83 4 45 34 2924 29004 239 393 32.4% 0.217 106 141 10 +MOT17-05-DPM 36.1% 62.7% 25.3% 35.6% 88.1% 133 11 49 73 333 4456 60 110 29.9% 0.247 65 20 25 +MOT17-05-FRCNN 44.8% 61.3% 35.3% 51.6% 89.6% 133 24 56 53 414 3347 63 69 44.7% 0.172 82 20 39 +MOT17-11-FRCNN 55.4% 74.2% 44.2% 55.3% 92.8% 75 15 34 26 405 4219 45 50 50.5% 0.096 35 22 12 +MOT17-05-SDP 43.1% 53.9% 35.9% 58.0% 87.0% 133 31 64 38 597 2905 84 128 48.2% 0.165 108 21 45 +MOT17-02-FRCNN 31.6% 52.8% 22.6% 34.6% 81.1% 62 6 25 31 1502 12148 93 127 26.0% 0.126 55 49 11 +MOT17-09-FRCNN 43.6% 61.2% 33.8% 53.4% 96.8% 26 6 16 4 95 2481 37 39 50.9% 0.096 27 16 6 +MOT17-10-FRCNN 40.0% 46.8% 35.0% 57.9% 77.6% 57 15 35 7 2150 5405 228 311 39.4% 0.162 147 89 16 +MOT17-04-FRCNN 49.5% 67.8% 39.0% 53.2% 92.4% 83 17 43 23 2077 22271 93 90 48.6% 0.108 42 56 5 +MOT17-13-DPM 24.9% 69.5% 15.2% 19.1% 87.6% 110 8 25 77 316 9416 35 102 16.1% 0.272 25 25 15 +MOT17-13-SDP 60.3% 78.0% 49.1% 54.5% 86.6% 110 43 25 42 980 5294 72 199 45.5% 0.212 70 24 27 +MOT17-02-SDP 34.3% 46.8% 27.1% 44.7% 77.1% 62 9 34 19 2468 10275 176 353 30.5% 0.201 114 72 13 +MOT17-13-FRCNN 50.4% 61.1% 42.9% 55.9% 79.6% 110 35 46 29 1667 5129 200 259 39.9% 0.169 141 83 34 +MOT17-02-DPM 22.8% 64.0% 13.9% 18.8% 86.6% 62 3 15 44 541 15090 47 103 15.6% 0.247 23 29 5 +MOT17-10-SDP 48.9% 56.9% 42.8% 67.4% 89.5% 57 23 30 4 1010 4189 163 290 58.2% 0.205 108 60 9 +MOT17-09-DPM 42.3% 52.8% 35.3% 53.3% 79.6% 26 2 19 5 725 2489 72 164 38.3% 0.273 50 23 5 +MOT17-09-SDP 48.5% 63.5% 39.2% 57.6% 93.2% 26 7 17 2 222 2260 46 78 52.5% 0.150 35 18 7 +MOT17-10-DPM 29.4% 49.6% 20.9% 36.1% 85.9% 57 7 18 32 762 8198 77 141 29.6% 0.251 36 48 8 +MOT17-11-DPM 50.3% 72.6% 38.5% 49.7% 93.6% 75 9 25 41 323 4748 45 58 45.8% 0.219 31 26 12 +MOT17-11-SDP 53.2% 62.9% 46.2% 66.5% 90.7% 75 19 39 17 646 3161 55 100 59.1% 0.149 40 28 15 +OVERALL 44.7% 62.6% 34.7% 49.2% 88.7% 1638 326 699 613 21139 171222 2051 3531 42.3% 0.170 1422 916 331 ``` diff --git a/main.js b/main.js index fc98f1f..1f12d8a 100755 --- a/main.js +++ b/main.js @@ -143,11 +143,13 @@ fs.readFile(`${pathRawDetectionsInput}`, function (err, f) { confidence: parseFloat(detectionOfThisFrameArray[6]) * 100, name: "" } - // If it's the first object for this frame, init empty array - if (!detections[detectionFrameIndex]) { - detections[detectionFrameIndex] = [] + if (detection.confidence > 0) { + // If it's the first object for this frame, init empty array + if (!detections[detectionFrameIndex]) { + detections[detectionFrameIndex] = [] + } + detections[detectionFrameIndex].push(detection); } - detections[detectionFrameIndex].push(detection); } }); } @@ -247,6 +249,16 @@ fs.readFile(`${pathRawDetectionsInput}`, function (err, f) { } }); }); + } else { + // Compute the lengths of all trajectories + const count = MOToutput.reduce((acc, e) => { + var id = e.split(',')[1]; + return acc.set(id, (acc.get(id) || 0) + 1); + }, new Map()); + + // MOT detections have a lot of false positives, so it's better to remove + // short trajectories, which are likely due to false positives + MOToutput = MOToutput.filter(line => count.get(line.split(',')[1]) >= 10); } if (!MODE_MOTChallenge) { diff --git a/package.json b/package.json index ee440c2..806ae3b 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "dependencies": { "lodash.isequal": "^4.5.0", "minimist": "^1.2.0", + "munkres-js": "^1.2.2", "uuid": "^3.2.1" }, "devDependencies": { diff --git a/tracker.js b/tracker.js index 8c7f762..d51fa3c 100644 --- a/tracker.js +++ b/tracker.js @@ -3,6 +3,7 @@ var ItemTracked = itemTrackedModule.ItemTracked; var kdTree = require('./lib/kdTree-min.js').kdTree; var isEqual = require('lodash.isequal') var iouAreas = require('./utils').iouAreas +var munkres = require('munkres-js'); var DEBUG_MODE = false; @@ -39,7 +40,10 @@ const params = { distanceFunc: iouDistance, // The distance limit for matching. If values need to be excluded from // matching set their distance to something greater than the distance limit - distanceLimit: 10000 + distanceLimit: 10000, + // The algorithm used to match tracks with new detections. Can be either + // 'kdTree' or 'munkres'. + matchingAlgorithm: 'kdTree', } // A dictionary of itemTracked currently tracked @@ -84,96 +88,136 @@ exports.updateTrackedItemsWithNewFrame = function(detectionsOfThisFrame, frameNb // Match existing Tracked items with the items detected in the new frame // For each look in the new detection to find the closest match if(detectionsOfThisFrame.length > 0) { - mapOfItemsTracked.forEach(function(itemTracked) { - - // First predict the new position of the itemTracked - var predictedPosition = itemTracked.predictNextPosition() - - // Make available for matching - itemTracked.makeAvailable(); - - // Search for a detection that matches - var treeSearchResult = treeDetectionsOfThisFrame.nearest(predictedPosition, 1, params.distanceLimit)[0]; - - // Only for debug assessments of predictions - var treeSearchResultWithoutPrediction = treeDetectionsOfThisFrame.nearest(itemTracked, 1, params.distanceLimit)[0]; - // Only if we enable the extra refinement - var treeSearchMultipleResults = treeDetectionsOfThisFrame.nearest(predictedPosition, 2, params.distanceLimit); - - // If we have found something - if(treeSearchResult) { - - // This is an extra refinement that happens in 0.001% of tracked items matching - // If IOU overlap is super similar for two potential match, add an extra check - // if(treeSearchMultipleResults.length === 2) { - - // var indexFirstChoice = 0; - // if(treeSearchMultipleResults[0][1] > treeSearchMultipleResults[1][1]) { - // indexFirstChoice = 1; - // } - - // var detectionFirstChoice = { - // bbox: treeSearchMultipleResults[indexFirstChoice][0], - // distance: treeSearchMultipleResults[indexFirstChoice][1] - // } - - // var detectionSecondChoice = { - // bbox: treeSearchMultipleResults[1 - indexFirstChoice][0], - // distance: treeSearchMultipleResults[1 - indexFirstChoice][1] - // } - - // const deltaDistance = Math.abs(detectionFirstChoice.distance - detectionSecondChoice.distance); - - // if(deltaDistance < 0.05) { - - // detectionFirstChoice.area = detectionFirstChoice.bbox.w * detectionFirstChoice.bbox.h; - // detectionSecondChoice.area = detectionSecondChoice.bbox.w * detectionSecondChoice.bbox.h; - // var itemTrackedArea = itemTracked.w * itemTracked.h; - - // var deltaAreaFirstChoice = Math.abs(detectionFirstChoice.area - itemTrackedArea) / (detectionFirstChoice.area + itemTrackedArea); - // var deltaAreaSecondChoice = Math.abs(detectionSecondChoice.area - itemTrackedArea) / (detectionSecondChoice.area + itemTrackedArea); - - // // Compare the area of each, priorize the detections that as a overal similar area - // // even if it overlaps less - // if(deltaAreaFirstChoice > deltaAreaSecondChoice) { - // if(Math.abs(deltaAreaFirstChoice - deltaAreaSecondChoice) > 0.5) { - // if(DEBUG_MODE) { - // console.log('Switch choice ! wise it seems different for frame: ' + frameNb + ' itemTracked ' + itemTracked.idDisplay) - // console.log(Math.abs(deltaAreaFirstChoice - deltaAreaSecondChoice)); - // } - // // Change tree search result: - // treeSearchResult = treeSearchMultipleResults[1 - indexFirstChoice] - // } - // } - // } - // } - - if(DEBUG_MODE) { - // Assess different results between predition or not - if(!isEqual(treeSearchResult[0], treeSearchResultWithoutPrediction && treeSearchResultWithoutPrediction[0])) { - console.log('Making the pre-prediction led to a difference result:'); - console.log('For frame ' + frameNb + ' itemNb ' + itemTracked.idDisplay) + if (params.matchingAlgorithm === 'munkres') { + var trackedItemIds = Array.from(mapOfItemsTracked.keys()); + + var costMatrix = Array.from(mapOfItemsTracked.values()). + map(itemTracked => { + var predictedPosition = itemTracked.predictNextPosition(); + return detectionsOfThisFrame.map( + detection => params.distanceFunc(predictedPosition, detection)); + }); + + mapOfItemsTracked.forEach(function(itemTracked) { + itemTracked.makeAvailable(); + }); + + munkres(costMatrix). + filter(m => costMatrix[m[0]][m[1]] <= params.distanceLimit). + forEach(m => { + var itemTracked = mapOfItemsTracked.get(trackedItemIds[m[0]]); + var updatedTrackedItemProperties = detectionsOfThisFrame[m[1]]; + matchedList[m[1]] = { idDisplay: itemTracked.idDisplay }; + itemTracked. + makeUnavailable(). + update(updatedTrackedItemProperties, frameNb); + }); + + matchedList.forEach(function(matched, index) { + if (!matched) { + if (Math.min(...costMatrix.map(m => m[index])) > params.distanceLimit) { + var newItemTracked = ItemTracked(detectionsOfThisFrame[index], frameNb, params.unMatchedFramesTolerance, params.fastDelete) + mapOfItemsTracked.set(newItemTracked.id, newItemTracked) + newItemTracked.makeUnavailable(); + costMatrix.push(detectionsOfThisFrame.map( + detection => params.distanceFunc(newItemTracked, detection))); } } + }); + } else if (params.matchingAlgorithm === 'kdTree') { + mapOfItemsTracked.forEach(function(itemTracked) { + + // First predict the new position of the itemTracked + var predictedPosition = itemTracked.predictNextPosition() + + // Make available for matching + itemTracked.makeAvailable(); + + // Search for a detection that matches + var treeSearchResult = treeDetectionsOfThisFrame.nearest(predictedPosition, 1, params.distanceLimit)[0]; + + // Only for debug assessments of predictions + var treeSearchResultWithoutPrediction = treeDetectionsOfThisFrame.nearest(itemTracked, 1, params.distanceLimit)[0]; + // Only if we enable the extra refinement + var treeSearchMultipleResults = treeDetectionsOfThisFrame.nearest(predictedPosition, 2, params.distanceLimit); + + // If we have found something + if(treeSearchResult) { + + // This is an extra refinement that happens in 0.001% of tracked items matching + // If IOU overlap is super similar for two potential match, add an extra check + // if(treeSearchMultipleResults.length === 2) { + + // var indexFirstChoice = 0; + // if(treeSearchMultipleResults[0][1] > treeSearchMultipleResults[1][1]) { + // indexFirstChoice = 1; + // } + + // var detectionFirstChoice = { + // bbox: treeSearchMultipleResults[indexFirstChoice][0], + // distance: treeSearchMultipleResults[indexFirstChoice][1] + // } + + // var detectionSecondChoice = { + // bbox: treeSearchMultipleResults[1 - indexFirstChoice][0], + // distance: treeSearchMultipleResults[1 - indexFirstChoice][1] + // } + + // const deltaDistance = Math.abs(detectionFirstChoice.distance - detectionSecondChoice.distance); + + // if(deltaDistance < 0.05) { + + // detectionFirstChoice.area = detectionFirstChoice.bbox.w * detectionFirstChoice.bbox.h; + // detectionSecondChoice.area = detectionSecondChoice.bbox.w * detectionSecondChoice.bbox.h; + // var itemTrackedArea = itemTracked.w * itemTracked.h; + + // var deltaAreaFirstChoice = Math.abs(detectionFirstChoice.area - itemTrackedArea) / (detectionFirstChoice.area + itemTrackedArea); + // var deltaAreaSecondChoice = Math.abs(detectionSecondChoice.area - itemTrackedArea) / (detectionSecondChoice.area + itemTrackedArea); + + // // Compare the area of each, priorize the detections that as a overal similar area + // // even if it overlaps less + // if(deltaAreaFirstChoice > deltaAreaSecondChoice) { + // if(Math.abs(deltaAreaFirstChoice - deltaAreaSecondChoice) > 0.5) { + // if(DEBUG_MODE) { + // console.log('Switch choice ! wise it seems different for frame: ' + frameNb + ' itemTracked ' + itemTracked.idDisplay) + // console.log(Math.abs(deltaAreaFirstChoice - deltaAreaSecondChoice)); + // } + // // Change tree search result: + // treeSearchResult = treeSearchMultipleResults[1 - indexFirstChoice] + // } + // } + // } + // } + + if(DEBUG_MODE) { + // Assess different results between predition or not + if(!isEqual(treeSearchResult[0], treeSearchResultWithoutPrediction && treeSearchResultWithoutPrediction[0])) { + console.log('Making the pre-prediction led to a difference result:'); + console.log('For frame ' + frameNb + ' itemNb ' + itemTracked.idDisplay) + } + } - var indexClosestNewDetectedItem = detectionsOfThisFrame.indexOf(treeSearchResult[0]); - // If this detections was not already matched to a tracked item - // (otherwise it would be matched to two tracked items...) - if(!matchedList[indexClosestNewDetectedItem]) { - matchedList[indexClosestNewDetectedItem] = { - idDisplay: itemTracked.idDisplay + var indexClosestNewDetectedItem = detectionsOfThisFrame.indexOf(treeSearchResult[0]); + // If this detections was not already matched to a tracked item + // (otherwise it would be matched to two tracked items...) + if(!matchedList[indexClosestNewDetectedItem]) { + matchedList[indexClosestNewDetectedItem] = { + idDisplay: itemTracked.idDisplay + } + // Update properties of tracked object + var updatedTrackedItemProperties = detectionsOfThisFrame[indexClosestNewDetectedItem] + mapOfItemsTracked.get(itemTracked.id) + .makeUnavailable() + .update(updatedTrackedItemProperties, frameNb) + } else { + // Means two already tracked item are concurrent to get assigned a new detections + // Rule is to priorize the oldest one to avoid id-reassignment } - // Update properties of tracked object - var updatedTrackedItemProperties = detectionsOfThisFrame[indexClosestNewDetectedItem] - mapOfItemsTracked.get(itemTracked.id) - .makeUnavailable() - .update(updatedTrackedItemProperties, frameNb) - } else { - // Means two already tracked item are concurrent to get assigned a new detections - // Rule is to priorize the oldest one to avoid id-reassignment } - } - }); + }); + } else { + throw `Unknown matching algorithm "${params.matchingAlgorithm}"`; + } } else { if(DEBUG_MODE) { console.log('[Tracker] Nothing detected for frame nº' + frameNb) @@ -184,32 +228,34 @@ exports.updateTrackedItemsWithNewFrame = function(detectionsOfThisFrame, frameNb }); } - // Add any unmatched items as new trackedItem only if those new items are not too similar - // to existing trackedItems this avoids adding some double match of YOLO and bring down drasticly reassignments - if(mapOfItemsTracked.size > 0) { // Safety check to see if we still have object tracked (could have been deleted previously) - // Rebuild tracked item tree to take in account the new positions - treeItemsTracked = new kdTree(Array.from(mapOfItemsTracked.values()), params.distanceFunc, ["x", "y", "w", "h"]); - // console.log(`Nb new items Unmatched : ${matchedList.filter((isMatched) => isMatched === false).length}`) - matchedList.forEach(function(matched, index) { - // Iterate through unmatched new detections - if(!matched) { - // Do not add as new tracked item if it is to similar to an existing one - var treeSearchResult = treeItemsTracked.nearest(detectionsOfThisFrame[index], 1, params.distanceLimit)[0]; - - if(!treeSearchResult) { - var newItemTracked = ItemTracked(detectionsOfThisFrame[index], frameNb, params.unMatchedFramesTolerance, params.fastDelete) - // Add it to the map - mapOfItemsTracked.set(newItemTracked.id, newItemTracked) - // Add it to the kd tree - treeItemsTracked.insert(newItemTracked); - // Make unvailable - newItemTracked.makeUnavailable(); - } else { - // console.log('Do not add, its overlapping an existing object') + if (params.matchingAlgorithm === 'kdTree') { + // Add any unmatched items as new trackedItem only if those new items are not too similar + // to existing trackedItems this avoids adding some double match of YOLO and bring down drasticly reassignments + if(mapOfItemsTracked.size > 0) { // Safety check to see if we still have object tracked (could have been deleted previously) + // Rebuild tracked item tree to take in account the new positions + treeItemsTracked = new kdTree(Array.from(mapOfItemsTracked.values()), params.distanceFunc, ["x", "y", "w", "h"]); + // console.log(`Nb new items Unmatched : ${matchedList.filter((isMatched) => isMatched === false).length}`) + matchedList.forEach(function(matched, index) { + // Iterate through unmatched new detections + if(!matched) { + // Do not add as new tracked item if it is to similar to an existing one + var treeSearchResult = treeItemsTracked.nearest(detectionsOfThisFrame[index], 1, params.distanceLimit)[0]; + + if(!treeSearchResult) { + var newItemTracked = ItemTracked(detectionsOfThisFrame[index], frameNb, params.unMatchedFramesTolerance, params.fastDelete) + // Add it to the map + mapOfItemsTracked.set(newItemTracked.id, newItemTracked) + // Add it to the kd tree + treeItemsTracked.insert(newItemTracked); + // Make unvailable + newItemTracked.makeUnavailable(); + } else { + // console.log('Do not add, its overlapping an existing object') + } } - } - }); - } + }); + } + } // Start killing the itemTracked (and predicting next position) // that are tracked but haven't been matched this frame