SSD Detector works on all frame area

Detector/DNNDetector.cpp

@@ -1,4 +1,5 @@
 #include "DNNDetector.h"
+#include "nms.h"
 
 ///
 /// \brief DNNDetector::DNNDetector
@@ -10,8 +11,18 @@ DNNDetector::DNNDetector(
     cv::UMat& colorFrame
 	)
     :
-      BaseDetector(collectPoints, colorFrame)
+      BaseDetector(collectPoints, colorFrame),
+      m_WHRatio(InWidth / (float)InHeight),
+      m_inScaleFactor(0.007843f),
+      m_meanVal(127.5),
+      m_confidenceThreshold(0.2f)
 {
+    m_classNames = { "background",
+                     "aeroplane", "bicycle", "bird", "boat",
+                     "bottle", "bus", "car", "cat", "chair",
+                     "cow", "diningtable", "dog", "horse",
+                     "motorbike", "person", "pottedplant",
+                     "sheep", "sofa", "train", "tvmonitor" };
 }
 
 ///
@@ -38,38 +49,63 @@ bool DNNDetector::Init(std::string modelConfiguration, std::string modelBinary)
 ///
 void DNNDetector::Detect(cv::UMat& colorFrame)
 {
-    const int inWidth = 300;
-    const int inHeight = 300;
-    const float WHRatio = inWidth / (float)inHeight;
-    const float inScaleFactor = 0.007843f;
-    const float meanVal = 127.5;
-    const float confidenceThreshold = 0.2f;
-    std::string classNames[] = {"background",
-                                "aeroplane", "bicycle", "bird", "boat",
-                                "bottle", "bus", "car", "cat", "chair",
-                                "cow", "diningtable", "dog", "horse",
-                                "motorbike", "person", "pottedplant",
-                                "sheep", "sofa", "train", "tvmonitor"};
-
-    cv::Size cropSize;
-    if (colorFrame.cols / (float)colorFrame.rows > WHRatio)
+    m_regions.clear();
+
+    regions_t tmpRegions;
+
+    cv::Mat colorMat = colorFrame.getMat(cv::ACCESS_READ);
+
+    if (colorFrame.cols / (float)colorFrame.rows > m_WHRatio)
     {
-        cropSize = cv::Size(cvRound(colorFrame.rows * WHRatio), colorFrame.rows);
+        cv::Rect crop(0, 0, cvRound(colorFrame.rows * m_WHRatio), colorFrame.rows);
+
+        for (; crop.x < colorMat.cols; crop.x += crop.width)
+        {
+            if (crop.x + crop.width >= colorMat.cols)
+            {
+                crop.x = colorMat.cols - crop.width;
+            }
+
+            DetectInCrop(colorMat, crop, tmpRegions);
+        }
     }
     else
     {
-        cropSize = cv::Size(colorFrame.cols, cvRound(colorFrame.cols / WHRatio));
+        cv::Rect crop(0, 0, colorFrame.cols, cvRound(colorFrame.cols / m_WHRatio));
+
+        for (; crop.y < colorMat.rows; crop.y += crop.height)
+        {
+            if (crop.y + crop.height >= colorMat.rows)
+            {
+                crop.y = colorMat.rows - crop.height;
+            }
+
+            DetectInCrop(colorMat, crop, tmpRegions);
+        }
     }
 
-    cv::Rect crop(cv::Point((colorFrame.cols - cropSize.width) / 2, (colorFrame.rows - cropSize.height) / 2), cropSize);
+    nms3<CRegion>(tmpRegions, m_regions, 0.5f,
+         [](const CRegion& reg) -> cv::Rect { return reg.m_rect; },
+    [](const CRegion& reg) -> float { return reg.m_confidence; },
+    0, 0.f);
+}
 
-    cv::Mat inputBlob = cv::dnn::blobFromImage(colorFrame.getMat(cv::ACCESS_READ), inScaleFactor, cv::Size(inWidth, inHeight), meanVal, false); //Convert Mat to batch of images
+///
+/// \brief DNNDetector::DetectInCrop
+/// \param colorFrame
+/// \param crop
+/// \param tmpRegions
+///
+void DNNDetector::DetectInCrop(cv::Mat colorFrame, const cv::Rect& crop, regions_t& tmpRegions)
+{
+    //Convert Mat to batch of images
+    cv::Mat inputBlob = cv::dnn::blobFromImage(cv::Mat(colorFrame, crop), m_inScaleFactor, cv::Size(InWidth, InHeight), m_meanVal, false, true);
 
     m_net.setInput(inputBlob, "data"); //set the network input
 
     cv::Mat detection = m_net.forward("detection_out"); //compute output
 
-    std::vector<double> layersTimings;
+    //std::vector<double> layersTimings;
     //double freq = cv::getTickFrequency() / 1000;
     //double time = m_net.getPerfProfile(layersTimings) / freq;
 
@@ -81,13 +117,13 @@ void DNNDetector::Detect(cv::UMat& colorFrame)
     //putText(frame, ss.str(), Point(20,20), 0, 0.5, Scalar(0,0,255));
     //std::cout << "Inference time, ms: " << time << endl;
 
-    m_regions.clear();
+    //cv::Point correctPoint((colorFrame.cols - crop.width) / 2, (colorFrame.rows - crop.height) / 2);
 
     for (int i = 0; i < detectionMat.rows; ++i)
     {
         float confidence = detectionMat.at<float>(i, 2);
 
-        if (confidence > confidenceThreshold)
+        if (confidence > m_confidenceThreshold)
         {
             size_t objectClass = (size_t)(detectionMat.at<float>(i, 1));
 
@@ -98,7 +134,7 @@ void DNNDetector::Detect(cv::UMat& colorFrame)
 
             cv::Rect object(xLeftBottom, yLeftBottom, xRightTop - xLeftBottom, yRightTop - yLeftBottom);
 
-            m_regions.push_back(CRegion(object, classNames[objectClass], confidence));
+            tmpRegions.push_back(CRegion(object, m_classNames[objectClass], confidence));
 
             //cv::rectangle(frame, object, Scalar(0, 255, 0));
             //std::string label = classNames[objectClass] + ": " + std::to_string(confidence);

Detector/DNNDetector.h

@@ -20,4 +20,14 @@ class DNNDetector : public BaseDetector
 
 private:
     cv::dnn::Net m_net;
+
+    void DetectInCrop(cv::Mat colorFrame, const cv::Rect& crop, regions_t& tmpRegions);
+
+    static const int InWidth = 300;
+    static const int InHeight = 300;
+    float m_WHRatio;
+    float m_inScaleFactor;
+    float m_meanVal;
+    float m_confidenceThreshold;
+    std::vector<std::string> m_classNames;
 };

TODO

@@ -1,4 +1,4 @@
 1. Collect points for all types of detectors
-2. MobileNet for all frame not for crop
-3. External API for the detectors options: for Background subtractors, confidenceThreshold for DNNDetector etc.
-4. New Multitarget tracking algorithm based on Discrete-Continuous Energy Minimization: https://bitbucket.org/amilan/dctracking
+2. External API for the detectors options: for Background subtractors, confidenceThreshold for DNNDetector etc.
+3. New Multitarget tracking algorithm based on Discrete-Continuous Energy Minimization: https://bitbucket.org/amilan/dctracking
+4. Deep SORT: https://github.com/nwojke/deep_sort

VideoExample.h

@@ -578,15 +578,15 @@ class DNNDetectorExample : public VideoExample
         m_detector->SetMinObjectSize(cv::Size(frame.cols / 20, frame.rows / 20));
 
         m_tracker = std::make_unique<CTracker>(m_useLocalTracking,
-                                               tracking::DistCenters,
+                                               tracking::DistRects,
                                                tracking::KalmanLinear,
                                                tracking::FilterRect,
                                                tracking::TrackKCF,      // Use KCF tracker for collisions resolving
                                                tracking::MatchHungrian,
                                                0.3f,                     // Delta time for Kalman filter
                                                0.1f,                     // Accel noise magnitude for Kalman filter
-                                               frame.rows / 5,          // Distance threshold between region and object on two frames
-                                               1 * m_fps,                // Maximum allowed skipped frames
+                                               frame.rows / 10,          // Distance threshold between region and object on two frames
+                                               2 * m_fps,                // Maximum allowed skipped frames
                                                5 * m_fps                 // Maximum trace length
                                                );
 
@@ -606,7 +606,7 @@ class DNNDetectorExample : public VideoExample
 
         for (const auto& track : m_tracker->tracks)
         {
-            if (track->IsRobust(2,                           // Minimal trajectory size
+            if (track->IsRobust(5,                           // Minimal trajectory size
                                 0.5f,                        // Minimal ratio raw_trajectory_points / trajectory_lenght
                                 cv::Size2f(0.1f, 8.0f))      // Min and max ratio: width / height
                     )

nms.h

@@ -145,3 +145,80 @@ inline void nms2(
         }
     }
 }
+
+
+/**
+ * @brief nms2
+ * Non maximum suppression with detection scores
+ * @param srcRects
+ * @param resRects
+ * @param thresh
+ * @param neighbors
+ */
+template<typename OBJ, typename GET_RECT_FUNC, typename GET_SCORE_FUNC>
+inline void nms3(
+        const std::vector<OBJ>& srcRects,
+        std::vector<OBJ>& resRects,
+        float thresh,
+        GET_RECT_FUNC GetRect,
+        GET_SCORE_FUNC GetScore,
+        int neighbors = 0,
+        float minScoresSum = 0.f
+        )
+{
+    resRects.clear();
+
+    const size_t size = srcRects.size();
+    if (!size)
+    {
+        return;
+    }
+
+    // Sort the bounding boxes by the detection score
+    std::multimap<float, size_t> idxs;
+    for (size_t i = 0; i < size; ++i)
+    {
+        idxs.insert(std::pair<float, size_t>(GetScore(srcRects[i]), i));
+    }
+
+    // keep looping while some indexes still remain in the indexes list
+    while (idxs.size() > 0)
+    {
+        // grab the last rectangle
+        auto lastElem = --std::end(idxs);
+        size_t lastPos = lastElem->second;
+        const cv::Rect& rect1 = GetRect(srcRects[lastPos]);
+
+        int neigborsCount = 0;
+        float scoresSum = lastElem->first;
+
+        idxs.erase(lastElem);
+
+        for (auto pos = std::begin(idxs); pos != std::end(idxs); )
+        {
+            // grab the current rectangle
+            const cv::Rect& rect2 = GetRect(srcRects[pos->second]);
+
+            float intArea = (rect1 & rect2).area();
+            float unionArea = rect1.area() + rect2.area() - intArea;
+            float overlap = intArea / unionArea;
+
+            // if there is sufficient overlap, suppress the current bounding box
+            if (overlap > thresh)
+            {
+                scoresSum += pos->first;
+                pos = idxs.erase(pos);
+                ++neigborsCount;
+            }
+            else
+            {
+                ++pos;
+            }
+        }
+        if (neigborsCount >= neighbors &&
+                scoresSum >= minScoresSum)
+        {
+            resRects.push_back(srcRects[lastPos]);
+        }
+    }
+}