diff --git a/Detector/DNNDetector.cpp b/Detector/DNNDetector.cpp index df71d8b05..605983d79 100644 --- a/Detector/DNNDetector.cpp +++ b/Detector/DNNDetector.cpp @@ -1,4 +1,5 @@ #include "DNNDetector.h" +#include "nms.h" /// /// \brief DNNDetector::DNNDetector @@ -10,8 +11,18 @@ DNNDetector::DNNDetector( cv::UMat& colorFrame ) : - BaseDetector(collectPoints, colorFrame) + BaseDetector(collectPoints, colorFrame), + m_WHRatio(InWidth / (float)InHeight), + m_inScaleFactor(0.007843f), + m_meanVal(127.5), + m_confidenceThreshold(0.2f) { + m_classNames = { "background", + "aeroplane", "bicycle", "bird", "boat", + "bottle", "bus", "car", "cat", "chair", + "cow", "diningtable", "dog", "horse", + "motorbike", "person", "pottedplant", + "sheep", "sofa", "train", "tvmonitor" }; } /// @@ -38,38 +49,63 @@ bool DNNDetector::Init(std::string modelConfiguration, std::string modelBinary) /// void DNNDetector::Detect(cv::UMat& colorFrame) { - const int inWidth = 300; - const int inHeight = 300; - const float WHRatio = inWidth / (float)inHeight; - const float inScaleFactor = 0.007843f; - const float meanVal = 127.5; - const float confidenceThreshold = 0.2f; - std::string classNames[] = {"background", - "aeroplane", "bicycle", "bird", "boat", - "bottle", "bus", "car", "cat", "chair", - "cow", "diningtable", "dog", "horse", - "motorbike", "person", "pottedplant", - "sheep", "sofa", "train", "tvmonitor"}; - - cv::Size cropSize; - if (colorFrame.cols / (float)colorFrame.rows > WHRatio) + m_regions.clear(); + + regions_t tmpRegions; + + cv::Mat colorMat = colorFrame.getMat(cv::ACCESS_READ); + + if (colorFrame.cols / (float)colorFrame.rows > m_WHRatio) { - cropSize = cv::Size(cvRound(colorFrame.rows * WHRatio), colorFrame.rows); + cv::Rect crop(0, 0, cvRound(colorFrame.rows * m_WHRatio), colorFrame.rows); + + for (; crop.x < colorMat.cols; crop.x += crop.width) + { + if (crop.x + crop.width >= colorMat.cols) + { + crop.x = colorMat.cols - crop.width; + } + + DetectInCrop(colorMat, crop, tmpRegions); + } } else { - cropSize = cv::Size(colorFrame.cols, cvRound(colorFrame.cols / WHRatio)); + cv::Rect crop(0, 0, colorFrame.cols, cvRound(colorFrame.cols / m_WHRatio)); + + for (; crop.y < colorMat.rows; crop.y += crop.height) + { + if (crop.y + crop.height >= colorMat.rows) + { + crop.y = colorMat.rows - crop.height; + } + + DetectInCrop(colorMat, crop, tmpRegions); + } } - cv::Rect crop(cv::Point((colorFrame.cols - cropSize.width) / 2, (colorFrame.rows - cropSize.height) / 2), cropSize); + nms3(tmpRegions, m_regions, 0.5f, + [](const CRegion& reg) -> cv::Rect { return reg.m_rect; }, + [](const CRegion& reg) -> float { return reg.m_confidence; }, + 0, 0.f); +} - cv::Mat inputBlob = cv::dnn::blobFromImage(colorFrame.getMat(cv::ACCESS_READ), inScaleFactor, cv::Size(inWidth, inHeight), meanVal, false); //Convert Mat to batch of images +/// +/// \brief DNNDetector::DetectInCrop +/// \param colorFrame +/// \param crop +/// \param tmpRegions +/// +void DNNDetector::DetectInCrop(cv::Mat colorFrame, const cv::Rect& crop, regions_t& tmpRegions) +{ + //Convert Mat to batch of images + cv::Mat inputBlob = cv::dnn::blobFromImage(cv::Mat(colorFrame, crop), m_inScaleFactor, cv::Size(InWidth, InHeight), m_meanVal, false, true); m_net.setInput(inputBlob, "data"); //set the network input cv::Mat detection = m_net.forward("detection_out"); //compute output - std::vector layersTimings; + //std::vector layersTimings; //double freq = cv::getTickFrequency() / 1000; //double time = m_net.getPerfProfile(layersTimings) / freq; @@ -81,13 +117,13 @@ void DNNDetector::Detect(cv::UMat& colorFrame) //putText(frame, ss.str(), Point(20,20), 0, 0.5, Scalar(0,0,255)); //std::cout << "Inference time, ms: " << time << endl; - m_regions.clear(); + //cv::Point correctPoint((colorFrame.cols - crop.width) / 2, (colorFrame.rows - crop.height) / 2); for (int i = 0; i < detectionMat.rows; ++i) { float confidence = detectionMat.at(i, 2); - if (confidence > confidenceThreshold) + if (confidence > m_confidenceThreshold) { size_t objectClass = (size_t)(detectionMat.at(i, 1)); @@ -98,7 +134,7 @@ void DNNDetector::Detect(cv::UMat& colorFrame) cv::Rect object(xLeftBottom, yLeftBottom, xRightTop - xLeftBottom, yRightTop - yLeftBottom); - m_regions.push_back(CRegion(object, classNames[objectClass], confidence)); + tmpRegions.push_back(CRegion(object, m_classNames[objectClass], confidence)); //cv::rectangle(frame, object, Scalar(0, 255, 0)); //std::string label = classNames[objectClass] + ": " + std::to_string(confidence); diff --git a/Detector/DNNDetector.h b/Detector/DNNDetector.h index 061c93b45..18e4292e6 100644 --- a/Detector/DNNDetector.h +++ b/Detector/DNNDetector.h @@ -20,4 +20,14 @@ class DNNDetector : public BaseDetector private: cv::dnn::Net m_net; + + void DetectInCrop(cv::Mat colorFrame, const cv::Rect& crop, regions_t& tmpRegions); + + static const int InWidth = 300; + static const int InHeight = 300; + float m_WHRatio; + float m_inScaleFactor; + float m_meanVal; + float m_confidenceThreshold; + std::vector m_classNames; }; diff --git a/TODO b/TODO index 430a7c2fb..93e53c643 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,4 @@ 1. Collect points for all types of detectors -2. MobileNet for all frame not for crop -3. External API for the detectors options: for Background subtractors, confidenceThreshold for DNNDetector etc. -4. New Multitarget tracking algorithm based on Discrete-Continuous Energy Minimization: https://bitbucket.org/amilan/dctracking +2. External API for the detectors options: for Background subtractors, confidenceThreshold for DNNDetector etc. +3. New Multitarget tracking algorithm based on Discrete-Continuous Energy Minimization: https://bitbucket.org/amilan/dctracking +4. Deep SORT: https://github.com/nwojke/deep_sort diff --git a/VideoExample.h b/VideoExample.h index 7c037bd12..5671b5ab9 100644 --- a/VideoExample.h +++ b/VideoExample.h @@ -578,15 +578,15 @@ class DNNDetectorExample : public VideoExample m_detector->SetMinObjectSize(cv::Size(frame.cols / 20, frame.rows / 20)); m_tracker = std::make_unique(m_useLocalTracking, - tracking::DistCenters, + tracking::DistRects, tracking::KalmanLinear, tracking::FilterRect, tracking::TrackKCF, // Use KCF tracker for collisions resolving tracking::MatchHungrian, 0.3f, // Delta time for Kalman filter 0.1f, // Accel noise magnitude for Kalman filter - frame.rows / 5, // Distance threshold between region and object on two frames - 1 * m_fps, // Maximum allowed skipped frames + frame.rows / 10, // Distance threshold between region and object on two frames + 2 * m_fps, // Maximum allowed skipped frames 5 * m_fps // Maximum trace length ); @@ -606,7 +606,7 @@ class DNNDetectorExample : public VideoExample for (const auto& track : m_tracker->tracks) { - if (track->IsRobust(2, // Minimal trajectory size + if (track->IsRobust(5, // Minimal trajectory size 0.5f, // Minimal ratio raw_trajectory_points / trajectory_lenght cv::Size2f(0.1f, 8.0f)) // Min and max ratio: width / height ) diff --git a/nms.h b/nms.h index 09f57d139..ed1e267e5 100644 --- a/nms.h +++ b/nms.h @@ -145,3 +145,80 @@ inline void nms2( } } } + + +/** + * @brief nms2 + * Non maximum suppression with detection scores + * @param srcRects + * @param resRects + * @param thresh + * @param neighbors + */ +template +inline void nms3( + const std::vector& srcRects, + std::vector& resRects, + float thresh, + GET_RECT_FUNC GetRect, + GET_SCORE_FUNC GetScore, + int neighbors = 0, + float minScoresSum = 0.f + ) +{ + resRects.clear(); + + const size_t size = srcRects.size(); + if (!size) + { + return; + } + + // Sort the bounding boxes by the detection score + std::multimap idxs; + for (size_t i = 0; i < size; ++i) + { + idxs.insert(std::pair(GetScore(srcRects[i]), i)); + } + + // keep looping while some indexes still remain in the indexes list + while (idxs.size() > 0) + { + // grab the last rectangle + auto lastElem = --std::end(idxs); + size_t lastPos = lastElem->second; + const cv::Rect& rect1 = GetRect(srcRects[lastPos]); + + int neigborsCount = 0; + float scoresSum = lastElem->first; + + idxs.erase(lastElem); + + for (auto pos = std::begin(idxs); pos != std::end(idxs); ) + { + // grab the current rectangle + const cv::Rect& rect2 = GetRect(srcRects[pos->second]); + + float intArea = (rect1 & rect2).area(); + float unionArea = rect1.area() + rect2.area() - intArea; + float overlap = intArea / unionArea; + + // if there is sufficient overlap, suppress the current bounding box + if (overlap > thresh) + { + scoresSum += pos->first; + pos = idxs.erase(pos); + ++neigborsCount; + } + else + { + ++pos; + } + } + if (neigborsCount >= neighbors && + scoresSum >= minScoresSum) + { + resRects.push_back(srcRects[lastPos]); + } + } +}