From e5af25ea038263dbe0a48e3375c0c4bbb6bbae1d Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Tue, 27 Aug 2019 07:02:27 +0300 Subject: [PATCH 1/8] fixed warning with variable name --- src/Tracker/HungarianAlg/HungarianAlg.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Tracker/HungarianAlg/HungarianAlg.cpp b/src/Tracker/HungarianAlg/HungarianAlg.cpp index ab9a17c7f..de2834b3b 100644 --- a/src/Tracker/HungarianAlg/HungarianAlg.cpp +++ b/src/Tracker/HungarianAlg/HungarianAlg.cpp @@ -546,11 +546,11 @@ void AssignmentProblemSolver::assignmentsuboptimal1(assignments_t& assignment, t if (singleValidationFound) { - for (size_t row = 0; row < nOfRows; row++) - if ((nOfValidObservations[row] > 1) && distMatrix[row + nOfRows*col] != std::numeric_limits::max()) + for (size_t nestedRow = 0; nestedRow < nOfRows; nestedRow++) + if ((nOfValidObservations[nestedRow] > 1) && distMatrix[nestedRow + nOfRows*col] != std::numeric_limits::max()) { - distMatrix[row + nOfRows*col] = std::numeric_limits::max(); - nOfValidObservations[row] -= 1; + distMatrix[nestedRow + nOfRows*col] = std::numeric_limits::max(); + nOfValidObservations[nestedRow] -= 1; nOfValidTracks[col] -= 1; repeatSteps = true; } From 39e003635a76cf639dfb8811a6e8b1ea9607d508 Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Wed, 28 Aug 2019 23:35:05 +0300 Subject: [PATCH 2/8] Color frames as base, gray frames - optionally --- async_detector/AsyncDetector.cpp | 23 ++---- async_detector/AsyncDetector.h | 3 +- cars_counting/CarsCounting.cpp | 54 +++++++------- cars_counting/CarsCounting.h | 2 - example/MouseExample.h | 2 +- example/VideoExample.cpp | 106 +++++++++++++------------- example/VideoExample.h | 7 +- example/examples.h | 39 ++-------- src/Detector/BaseDetector.cpp | 26 +++---- src/Detector/BaseDetector.h | 5 ++ src/Detector/FaceDetector.h | 5 ++ src/Detector/MotionDetector.h | 5 ++ src/Detector/PedestrianDetector.h | 5 ++ src/Detector/SSDMobileNetDetector.h | 5 ++ src/Detector/YoloDarknetDetector.h | 5 ++ src/Detector/YoloDetector.h | 5 ++ src/Tracker/Ctracker.cpp | 112 ++++++++++++++-------------- src/Tracker/Ctracker.h | 52 +++++++++++-- src/Tracker/track.cpp | 59 ++++++++++++--- src/Tracker/track.h | 27 +++++-- src/defines.h | 25 ++++--- 21 files changed, 330 insertions(+), 242 deletions(-) diff --git a/async_detector/AsyncDetector.cpp b/async_detector/AsyncDetector.cpp index 97f9247fe..dee3b50c8 100644 --- a/async_detector/AsyncDetector.cpp +++ b/async_detector/AsyncDetector.cpp @@ -298,7 +298,7 @@ void AsyncDetector::CaptureThread(std::string fileName, int startFrame, float* f const int minStaticTime = 5; TrackerSettings trackerSettings; - trackerSettings.m_distType = tracking::DistCenters; + trackerSettings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); trackerSettings.m_kalmanType = tracking::KalmanLinear; trackerSettings.m_filterGoal = tracking::FilterRect; trackerSettings.m_lostTrackType = tracking::TrackKCF; // Use KCF tracker for collisions resolving @@ -323,11 +323,9 @@ void AsyncDetector::CaptureThread(std::string fileName, int startFrame, float* f // Capture the first frame cv::Mat firstFrame; - cv::UMat firstGray; capture >> firstFrame; - cv::cvtColor(firstFrame, firstGray, cv::COLOR_BGR2GRAY); - std::thread thDetection(DetectThread, detectorConfig, firstGray, framesQue, stopFlag); + std::thread thDetection(DetectThread, detectorConfig, firstFrame, framesQue, stopFlag); std::thread thTracking(TrackingThread, trackerSettings, framesQue, stopFlag); // Capture frame @@ -347,7 +345,6 @@ void AsyncDetector::CaptureThread(std::string fileName, int startFrame, float* f { frameInfo->m_clFrame = frameInfo->m_frame.getUMat(cv::ACCESS_READ); } - cv::cvtColor(frameInfo->m_frame, frameInfo->m_gray, cv::COLOR_BGR2GRAY); framesQue->AddNewFrame(frameInfo, 15); @@ -371,10 +368,11 @@ void AsyncDetector::CaptureThread(std::string fileName, int startFrame, float* f /// \brief AsyncDetector::DetectThread /// \param /// -void AsyncDetector::DetectThread(const config_t& config, cv::UMat firstGray, FramesQueue* framesQue, bool* stopFlag) +void AsyncDetector::DetectThread(const config_t& config, cv::Mat firstFrame, FramesQueue* framesQue, bool* stopFlag) { - std::unique_ptr detector = std::unique_ptr(CreateDetector(tracking::Detectors::Yolo_Darknet, config, firstGray)); - detector->SetMinObjectSize(cv::Size(firstGray.cols / 50, firstGray.cols / 50)); + cv::UMat ufirst = firstFrame.getUMat(cv::ACCESS_READ); + std::unique_ptr detector = std::unique_ptr(CreateDetector(tracking::Detectors::Yolo_Darknet, config, ufirst)); + detector->SetMinObjectSize(cv::Size(firstFrame.cols / 50, firstFrame.cols / 50)); for (; !(*stopFlag);) { @@ -406,14 +404,7 @@ void AsyncDetector::TrackingThread(const TrackerSettings& settings, FramesQueue* frame_ptr frameInfo = framesQue->GetFirstDetectedFrame(); if (frameInfo) { - if (tracker->GrayFrameToTrack()) - { - tracker->Update(frameInfo->m_regions, frameInfo->m_gray, frameInfo->m_fps); - } - else - { - tracker->Update(frameInfo->m_regions, frameInfo->m_clFrame, frameInfo->m_fps); - } + tracker->Update(frameInfo->m_regions, frameInfo->m_clFrame, frameInfo->m_fps); frameInfo->m_tracks = tracker->GetTracks(); frameInfo->m_inTracker = 2; diff --git a/async_detector/AsyncDetector.h b/async_detector/AsyncDetector.h index 700b2363a..b0c876d06 100644 --- a/async_detector/AsyncDetector.h +++ b/async_detector/AsyncDetector.h @@ -20,7 +20,6 @@ struct FrameInfo { cv::Mat m_frame; cv::UMat m_clFrame; - cv::UMat m_gray; regions_t m_regions; std::vector m_tracks; int64 m_dt = 0; @@ -63,6 +62,6 @@ class AsyncDetector void DrawTrack(cv::Mat frame, int resizeCoeff, const TrackingObject& track, bool drawTrajectory = true); static void CaptureThread(std::string fileName, int startFrame, float* fps, FramesQueue* framesQue, bool* stopFlag); - static void DetectThread(const config_t& config, cv::UMat firstGray, FramesQueue* framesQue, bool* stopFlag); + static void DetectThread(const config_t& config, cv::Mat firstFrame, FramesQueue* framesQue, bool* stopFlag); static void TrackingThread(const TrackerSettings& settings, FramesQueue* framesQue, bool* stopFlag); }; diff --git a/cars_counting/CarsCounting.cpp b/cars_counting/CarsCounting.cpp index 58ec0d8c2..28d84cf43 100644 --- a/cars_counting/CarsCounting.cpp +++ b/cars_counting/CarsCounting.cpp @@ -78,7 +78,23 @@ void CarsCounting::Process() m_fps = std::max(1.f, (float)capture.get(cv::CAP_PROP_FPS)); cv::Mat colorFrame; - cv::UMat grayFrame; + capture >> colorFrame; + if (colorFrame.empty()) + { + std::cerr << "Frame is empty!" << std::endl; + return; + } + if (!m_isTrackerInitialized) + { + cv::UMat uframe = colorFrame.getUMat(cv::ACCESS_READ); + m_isTrackerInitialized = InitTracker(uframe); + if (!m_isTrackerInitialized) + { + std::cerr << "Tracker initialize error!!!" << std::endl; + return; + } + } + for (;;) { capture >> colorFrame; @@ -87,31 +103,24 @@ void CarsCounting::Process() std::cerr << "Frame is empty!" << std::endl; break; } - cv::cvtColor(colorFrame, grayFrame, cv::COLOR_BGR2GRAY); - - if (!m_isTrackerInitialized) - { - m_isTrackerInitialized = InitTracker(grayFrame); - if (!m_isTrackerInitialized) - { - std::cerr << "Tracker initilize error!!!" << std::endl; - break; - } - } int64 t1 = cv::getTickCount(); - cv::UMat clFrame; - if (!GrayProcessing() || !m_tracker->GrayFrameToTrack()) + cv::UMat uframe; + if (!m_detector->CanGrayProcessing() || !m_tracker->GrayFrameToTrack()) { - clFrame = colorFrame.getUMat(cv::ACCESS_READ); + uframe = colorFrame.getUMat(cv::ACCESS_READ); } + else + { + cv::cvtColor(colorFrame, uframe, cv::COLOR_BGR2GRAY); + } - m_detector->Detect(GrayProcessing() ? grayFrame : clFrame); + m_detector->Detect(uframe); const regions_t& regions = m_detector->GetDetects(); - m_tracker->Update(regions, m_tracker->GrayFrameToTrack() ? grayFrame : clFrame, m_fps); + m_tracker->Update(regions, uframe, m_fps); int64 t2 = cv::getTickCount(); @@ -160,15 +169,6 @@ void CarsCounting::Process() #endif } -/// -/// \brief CarsCounting::GrayProcessing -/// \return -/// -bool CarsCounting::GrayProcessing() const -{ - return true; -} - /// /// \brief CarsCounting::DrawTrack /// \param frame @@ -259,7 +259,7 @@ bool CarsCounting::InitTracker(cv::UMat frame) m_detector->SetMinObjectSize(cv::Size(m_minObjWidth, m_minObjWidth)); TrackerSettings settings; - settings.m_distType = tracking::DistCenters; + settings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use KCF tracker for collisions resolving diff --git a/cars_counting/CarsCounting.h b/cars_counting/CarsCounting.h index eac4c2b72..bbb6a5b75 100644 --- a/cars_counting/CarsCounting.h +++ b/cars_counting/CarsCounting.h @@ -241,8 +241,6 @@ class CarsCounting bool m_showLogs = false; float m_fps = 0; - virtual bool GrayProcessing() const; - virtual bool InitTracker(cv::UMat frame); virtual void DrawData(cv::Mat frame, int framesCounter, int currTime); diff --git a/example/MouseExample.h b/example/MouseExample.h index 33fd42f23..e2eab7c92 100644 --- a/example/MouseExample.h +++ b/example/MouseExample.h @@ -47,7 +47,7 @@ void MouseTracking(cv::CommandLineParser parser) cv::setMouseCallback("Video", mv_MouseCallback, (void*)&pointXY); TrackerSettings settings; - settings.m_distType = tracking::DistCenters; + settings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterCenter; settings.m_lostTrackType = tracking::TrackNone; diff --git a/example/VideoExample.cpp b/example/VideoExample.cpp index a02ead256..e563dea42 100644 --- a/example/VideoExample.cpp +++ b/example/VideoExample.cpp @@ -52,7 +52,6 @@ void VideoExample::SyncProcess() #endif cv::Mat frame; - cv::UMat gray; double freq = cv::getTickFrequency(); int64 allTime = 0; @@ -75,32 +74,35 @@ void VideoExample::SyncProcess() { break; } - cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY); - if (!m_isDetectorInitialized) - { - m_isDetectorInitialized = InitDetector(gray); - if (!m_isDetectorInitialized) - { - std::cerr << "CaptureAndDetect: Detector initilize error!!!" << std::endl; - break; - } - } - if (!m_isTrackerInitialized) - { - m_isTrackerInitialized = InitTracker(gray); - if (!m_isTrackerInitialized) - { - std::cerr << "CaptureAndDetect: Tracker initilize error!!!" << std::endl; - break; - } - } + if (!m_isDetectorInitialized || !m_isTrackerInitialized) + { + cv::UMat ufirst = frame.getUMat(cv::ACCESS_READ); + if (!m_isDetectorInitialized) + { + m_isDetectorInitialized = InitDetector(ufirst); + if (!m_isDetectorInitialized) + { + std::cerr << "CaptureAndDetect: Detector initialize error!!!" << std::endl; + break; + } + } + if (!m_isTrackerInitialized) + { + m_isTrackerInitialized = InitTracker(ufirst); + if (!m_isTrackerInitialized) + { + std::cerr << "CaptureAndDetect: Tracker initialize error!!!" << std::endl; + break; + } + } + } int64 t1 = cv::getTickCount(); regions_t regions; - Detection(frame, gray, regions); - Tracking(frame, gray, regions); + Detection(frame, regions); + Tracking(frame, regions); int64 t2 = cv::getTickCount(); @@ -181,10 +183,11 @@ void VideoExample::AsyncProcess() if (!m_isTrackerInitialized) { - m_isTrackerInitialized = InitTracker(frameInfo.m_gray); + cv::UMat ufirst = frameInfo.m_frame.getUMat(cv::ACCESS_READ); + m_isTrackerInitialized = InitTracker(ufirst); if (!m_isTrackerInitialized) { - std::cerr << "CaptureAndDetect: Tracker initilize error!!!" << std::endl; + std::cerr << "CaptureAndDetect: Tracker initialize error!!!" << std::endl; frameInfo.m_cond.notify_one(); break; } @@ -192,7 +195,7 @@ void VideoExample::AsyncProcess() int64 t1 = cv::getTickCount(); - Tracking(frameInfo.m_frame, frameInfo.m_gray, frameInfo.m_regions); + Tracking(frameInfo.m_frame, frameInfo.m_regions); int64 t2 = cv::getTickCount(); @@ -291,21 +294,21 @@ void VideoExample::CaptureAndDetect(VideoExample* thisPtr, std::atomic& st frameInfo.m_cond.notify_one(); break; } - cv::cvtColor(frameInfo.m_frame, frameInfo.m_gray, cv::COLOR_BGR2GRAY); if (!thisPtr->m_isDetectorInitialized) { - thisPtr->m_isDetectorInitialized = thisPtr->InitDetector(frameInfo.m_gray); + cv::UMat ufirst = frameInfo.m_frame.getUMat(cv::ACCESS_READ); + thisPtr->m_isDetectorInitialized = thisPtr->InitDetector(ufirst); if (!thisPtr->m_isDetectorInitialized) { - std::cerr << "CaptureAndDetect: Detector initilize error!!!" << std::endl; + std::cerr << "CaptureAndDetect: Detector initialize error!!!" << std::endl; frameInfo.m_cond.notify_one(); break; } } int64 t1 = cv::getTickCount(); - thisPtr->Detection(frameInfo.m_frame, frameInfo.m_gray, frameInfo.m_regions); + thisPtr->Detection(frameInfo.m_frame, frameInfo.m_regions); int64 t2 = cv::getTickCount(); frameInfo.m_dt = t2 - t1; @@ -320,30 +323,24 @@ void VideoExample::CaptureAndDetect(VideoExample* thisPtr, std::atomic& st stopCapture = true; } -/// -/// \brief VideoExample::GrayProcessing -/// \return -/// -bool VideoExample::GrayProcessing() const -{ - return true; -} - /// /// \brief VideoExample::Detection /// \param frame -/// \param grayFrame /// \param regions /// -void VideoExample::Detection(cv::Mat frame, cv::UMat grayFrame, regions_t& regions) +void VideoExample::Detection(cv::Mat frame, regions_t& regions) { - cv::UMat clFrame; - if (!GrayProcessing() || !m_tracker->GrayFrameToTrack()) + cv::UMat uframe; + if (!m_detector->CanGrayProcessing()) { - clFrame = frame.getUMat(cv::ACCESS_READ); + uframe = frame.getUMat(cv::ACCESS_READ); } + else + { + cv::cvtColor(frame, uframe, cv::COLOR_BGR2GRAY); + } - m_detector->Detect(GrayProcessing() ? grayFrame : clFrame); + m_detector->Detect(uframe); const regions_t& regs = m_detector->GetDetects(); @@ -353,18 +350,21 @@ void VideoExample::Detection(cv::Mat frame, cv::UMat grayFrame, regions_t& regio /// /// \brief VideoExample::Tracking /// \param frame -/// \param grayFrame /// \param regions /// -void VideoExample::Tracking(cv::Mat frame, cv::UMat grayFrame, const regions_t& regions) +void VideoExample::Tracking(cv::Mat frame, const regions_t& regions) { - cv::UMat clFrame; - if (!GrayProcessing() || !m_tracker->GrayFrameToTrack()) - { - clFrame = frame.getUMat(cv::ACCESS_READ); - } - - m_tracker->Update(regions, m_tracker->GrayFrameToTrack() ? grayFrame : clFrame, m_fps); + cv::UMat uframe; + if (!m_tracker->GrayFrameToTrack()) + { + uframe = frame.getUMat(cv::ACCESS_READ); + } + else + { + cv::cvtColor(frame, uframe, cv::COLOR_BGR2GRAY); + } + + m_tracker->Update(regions, uframe, m_fps); } /// diff --git a/example/VideoExample.h b/example/VideoExample.h index 3f657d607..24ed7aaf1 100644 --- a/example/VideoExample.h +++ b/example/VideoExample.h @@ -41,13 +41,11 @@ class VideoExample static void CaptureAndDetect(VideoExample* thisPtr, std::atomic& stopCapture); - virtual bool GrayProcessing() const; - virtual bool InitDetector(cv::UMat frame) = 0; virtual bool InitTracker(cv::UMat frame) = 0; - void Detection(cv::Mat frame, cv::UMat grayFrame, regions_t& regions); - void Tracking(cv::Mat frame, cv::UMat grayFrame, const regions_t& regions); + void Detection(cv::Mat frame, regions_t& regions); + void Tracking(cv::Mat frame, const regions_t& regions); virtual void DrawData(cv::Mat frame, int framesCounter, int currTime) = 0; @@ -67,7 +65,6 @@ class VideoExample struct FrameInfo { cv::Mat m_frame; - cv::UMat m_gray; regions_t m_regions; int64 m_dt = 0; diff --git a/example/examples.h b/example/examples.h index 785072682..e0b5f29ae 100644 --- a/example/examples.h +++ b/example/examples.h @@ -60,7 +60,7 @@ class MotionDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.m_distType = tracking::DistCenters; + settings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving @@ -175,7 +175,7 @@ class FaceDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.m_distType = tracking::DistJaccard; + settings.SetDistances({ 0.0f, 0.0f, 1.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanUnscented; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -271,7 +271,7 @@ class PedestrianDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.m_distType = tracking::DistRects; + settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -368,7 +368,7 @@ class SSDMobileNetExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.m_distType = tracking::DistRects; + settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -424,15 +424,6 @@ class SSDMobileNetExample : public VideoExample m_detector->CalcMotionMap(frame); } - - /// - /// \brief GrayProcessing - /// \return - /// - bool GrayProcessing() const - { - return false; - } }; // ---------------------------------------------------------------------- @@ -501,7 +492,7 @@ class YoloExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.m_distType = tracking::DistRects; + settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -557,15 +548,6 @@ class YoloExample : public VideoExample m_detector->CalcMotionMap(frame); } - - /// - /// \brief GrayProcessing - /// \return - /// - bool GrayProcessing() const - { - return false; - } }; #ifdef BUILD_YOLO_LIB @@ -634,7 +616,7 @@ class YoloDarknetExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.m_distType = tracking::DistRects; + settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving @@ -690,15 +672,6 @@ class YoloDarknetExample : public VideoExample m_detector->CalcMotionMap(frame); } - - /// - /// \brief GrayProcessing - /// \return - /// - bool GrayProcessing() const - { - return false; - } }; #endif diff --git a/src/Detector/BaseDetector.cpp b/src/Detector/BaseDetector.cpp index 876196587..d2aa3b652 100644 --- a/src/Detector/BaseDetector.cpp +++ b/src/Detector/BaseDetector.cpp @@ -17,7 +17,7 @@ BaseDetector* CreateDetector( tracking::Detectors detectorType, const config_t& config, - cv::UMat& gray + cv::UMat& frame ) { BaseDetector* detector = nullptr; @@ -25,53 +25,53 @@ BaseDetector* CreateDetector( switch (detectorType) { case tracking::Motion_VIBE: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_VIBE, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_VIBE, frame); break; case tracking::Motion_MOG: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_MOG, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_MOG, frame); break; case tracking::Motion_GMG: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_GMG, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_GMG, frame); break; case tracking::Motion_CNT: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_CNT, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_CNT, frame); break; case tracking::Motion_SuBSENSE: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_SuBSENSE, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_SuBSENSE, frame); break; case tracking::Motion_LOBSTER: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_LOBSTER, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_LOBSTER, frame); break; case tracking::Motion_MOG2: - detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_MOG2, gray); + detector = new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_MOG2, frame); break; case tracking::Face_HAAR: - detector = new FaceDetector(gray); + detector = new FaceDetector(frame); break; case tracking::Pedestrian_HOG: case tracking::Pedestrian_C4: - detector = new PedestrianDetector(gray); + detector = new PedestrianDetector(frame); break; case tracking::SSD_MobileNet: - detector = new SSDMobileNetDetector(gray); + detector = new SSDMobileNetDetector(frame); break; case tracking::Yolo_OCV: - detector = new YoloOCVDetector(gray); + detector = new YoloOCVDetector(frame); break; case tracking::Yolo_Darknet: #ifdef BUILD_YOLO_LIB - detector = new YoloDarknetDetector(gray); + detector = new YoloDarknetDetector(frame); #else std::cerr << "Darknet inference engine was not configured in CMake" << std::endl; #endif diff --git a/src/Detector/BaseDetector.h b/src/Detector/BaseDetector.h index d0158db31..97f18937d 100644 --- a/src/Detector/BaseDetector.h +++ b/src/Detector/BaseDetector.h @@ -37,6 +37,11 @@ class BaseDetector /// virtual void Detect(cv::UMat& frame) = 0; + /// + /// \brief CanGrayProcessing + /// + virtual bool CanGrayProcessing() const = 0; + /// /// \brief SetMinObjectSize /// \param minObjectSize diff --git a/src/Detector/FaceDetector.h b/src/Detector/FaceDetector.h index 7af7ab72d..ea6713890 100644 --- a/src/Detector/FaceDetector.h +++ b/src/Detector/FaceDetector.h @@ -15,6 +15,11 @@ class FaceDetector : public BaseDetector void Detect(cv::UMat& gray); + bool CanGrayProcessing() const + { + return true; + } + private: cv::CascadeClassifier m_cascade; }; diff --git a/src/Detector/MotionDetector.h b/src/Detector/MotionDetector.h index 7d5f50378..d03c3bd43 100644 --- a/src/Detector/MotionDetector.h +++ b/src/Detector/MotionDetector.h @@ -16,6 +16,11 @@ class MotionDetector : public BaseDetector void Detect(cv::UMat& gray); + bool CanGrayProcessing() const + { + return true; + } + void CalcMotionMap(cv::Mat frame); private: diff --git a/src/Detector/PedestrianDetector.h b/src/Detector/PedestrianDetector.h index 414bae9a9..9536beb3a 100644 --- a/src/Detector/PedestrianDetector.h +++ b/src/Detector/PedestrianDetector.h @@ -22,6 +22,11 @@ class PedestrianDetector : public BaseDetector void Detect(cv::UMat& gray); + bool CanGrayProcessing() const + { + return true; + } + private: DetectorTypes m_detectorType; diff --git a/src/Detector/SSDMobileNetDetector.h b/src/Detector/SSDMobileNetDetector.h index c3b7e2d42..6ce5ec8bc 100644 --- a/src/Detector/SSDMobileNetDetector.h +++ b/src/Detector/SSDMobileNetDetector.h @@ -22,6 +22,11 @@ class SSDMobileNetDetector : public BaseDetector void Detect(cv::UMat& colorFrame); + bool CanGrayProcessing() const + { + return false; + } + private: cv::dnn::Net m_net; diff --git a/src/Detector/YoloDarknetDetector.h b/src/Detector/YoloDarknetDetector.h index e66c6ad6f..a0707c7c7 100644 --- a/src/Detector/YoloDarknetDetector.h +++ b/src/Detector/YoloDarknetDetector.h @@ -24,6 +24,11 @@ class YoloDarknetDetector : public BaseDetector void Detect(cv::UMat& colorFrame); + bool CanGrayProcessing() const + { + return false; + } + private: std::unique_ptr m_detector; diff --git a/src/Detector/YoloDetector.h b/src/Detector/YoloDetector.h index 05c5676ee..e027fb7c4 100644 --- a/src/Detector/YoloDetector.h +++ b/src/Detector/YoloDetector.h @@ -23,6 +23,11 @@ class YoloOCVDetector : public BaseDetector void Detect(cv::UMat& colorFrame); + bool CanGrayProcessing() const + { + return false; + } + private: cv::dnn::Net m_net; diff --git a/src/Tracker/Ctracker.cpp b/src/Tracker/Ctracker.cpp index c68a3068c..2780b1b34 100644 --- a/src/Tracker/Ctracker.cpp +++ b/src/Tracker/Ctracker.cpp @@ -27,29 +27,29 @@ CTracker::~CTracker(void) /// /// \brief CTracker::Update /// \param regions -/// \param grayFrame +/// \param currFrame /// \param fps /// void CTracker::Update( const regions_t& regions, - cv::UMat grayFrame, + cv::UMat currFrame, float fps ) { - UpdateTrackingState(regions, grayFrame, fps); + UpdateTrackingState(regions, currFrame, fps); - grayFrame.copyTo(m_prevFrame); + currFrame.copyTo(m_prevFrame); } /// /// \brief CTracker::UpdateTrackingState /// \param regions -/// \param grayFrame +/// \param currFrame /// \param fps /// void CTracker::UpdateTrackingState( const regions_t& regions, - cv::UMat grayFrame, + cv::UMat currFrame, float fps ) { @@ -62,9 +62,9 @@ void CTracker::UpdateTrackingState( { // Distance matrix between all tracks to all regions distMatrix_t costMatrix(N * M); - const track_t maxPossibleCost = static_cast(grayFrame.cols * grayFrame.rows); + const track_t maxPossibleCost = static_cast(currFrame.cols * currFrame.rows); track_t maxCost = 0; - CreateDistaceMatrix(regions, costMatrix, maxPossibleCost, maxCost); + CreateDistaceMatrix(regions, costMatrix, maxPossibleCost, maxCost, currFrame); // Solving assignment problem (tracks and predictions of Kalman filter) if (m_settings.m_matchType == tracking::MatchHungrian) @@ -134,12 +134,12 @@ void CTracker::UpdateTrackingState( m_tracks[i]->Update( regions[assignment[i]], true, m_settings.m_maxTraceLength, - m_prevFrame, grayFrame, + m_prevFrame, currFrame, m_settings.m_useAbandonedDetection ? cvRound(m_settings.m_minStaticTime * fps) : 0); } else // if not continue using predictions { - m_tracks[i]->Update(CRegion(), false, m_settings.m_maxTraceLength, m_prevFrame, grayFrame, 0); + m_tracks[i]->Update(CRegion(), false, m_settings.m_maxTraceLength, m_prevFrame, currFrame, 0); } } } @@ -151,57 +151,57 @@ void CTracker::UpdateTrackingState( /// \param maxPossibleCost /// \param maxCost /// -void CTracker::CreateDistaceMatrix(const regions_t& regions, distMatrix_t& costMatrix, track_t maxPossibleCost, track_t& maxCost) +void CTracker::CreateDistaceMatrix(const regions_t& regions, distMatrix_t& costMatrix, track_t maxPossibleCost, track_t& maxCost, cv::UMat currFrame) { const size_t N = m_tracks.size(); // Tracking objects maxCost = 0; - switch (m_settings.m_distType) - { - case tracking::DistCenters: - for (size_t i = 0; i < m_tracks.size(); i++) - { - for (size_t j = 0; j < regions.size(); j++) - { - auto dist = m_tracks[i]->CheckType(regions[j].m_type) ? m_tracks[i]->CalcDist(regions[j].m_rrect.center) : maxPossibleCost; - costMatrix[i + j * N] = dist; - if (dist > maxCost) - { - maxCost = dist; - } - } - } - break; - case tracking::DistRects: - for (size_t i = 0; i < m_tracks.size(); i++) - { - for (size_t j = 0; j < regions.size(); j++) - { - auto dist = m_tracks[i]->CheckType(regions[j].m_type) ? m_tracks[i]->CalcDist(regions[j]) : maxPossibleCost; - costMatrix[i + j * N] = dist; - if (dist > maxCost) - { - maxCost = dist; - } - } - } - break; + for (size_t i = 0; i < m_tracks.size(); i++) + { + const auto& track = m_tracks[i]; - case tracking::DistJaccard: - for (size_t i = 0; i < m_tracks.size(); i++) - { - for (size_t j = 0; j < regions.size(); j++) - { - auto dist = m_tracks[i]->CheckType(regions[j].m_type) ? m_tracks[i]->CalcDistJaccard(regions[j]) : 1; - costMatrix[i + j * N] = dist; - if (dist > maxCost) - { - maxCost = dist; - } - } - } - break; - } + for (size_t j = 0; j < regions.size(); j++) + { + auto dist = maxPossibleCost; + if (m_tracks[i]->CheckType(regions[j].m_type)) + { + dist = 0; + size_t ind = 0; + if (m_settings.m_distType[ind] > 0.0f && ind == tracking::DistCenters) + { + dist += m_settings.m_distType[ind] * track->CalcDistCenter(regions[j]); + } + ++ind; + if (m_settings.m_distType[ind] > 0.0f && ind == tracking::DistRects) + { + dist += m_settings.m_distType[ind] * track->CalcDistRect(regions[j]); + } + ++ind; + if (m_settings.m_distType[ind] > 0.0f && ind == tracking::DistJaccard) + { + dist += m_settings.m_distType[ind] * track->CalcDistJaccard(regions[j]); + } + ++ind; + if (m_settings.m_distType[ind] > 0.0f && ind == tracking::DistHist) + { + dist += m_settings.m_distType[ind] * track->CalcDistHist(regions[j], currFrame); + } + ++ind; + if (m_settings.m_distType[ind] > 0.0f && ind == tracking::DistHOG) + { + dist += m_settings.m_distType[ind] * track->CalcDistHOG(regions[j]); + } + ++ind; + assert(ind == tracking::DistsCount); + } + + costMatrix[i + j * N] = dist; + if (dist > maxCost) + { + maxCost = dist; + } + } + } } /// diff --git a/src/Tracker/Ctracker.h b/src/Tracker/Ctracker.h index ba6476206..e79f033e3 100644 --- a/src/Tracker/Ctracker.h +++ b/src/Tracker/Ctracker.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "defines.h" #include "track.h" @@ -16,12 +17,14 @@ /// struct TrackerSettings { - tracking::DistType m_distType = tracking::DistCenters; + //tracking::DistType m_distType = tracking::DistCenters; tracking::KalmanType m_kalmanType = tracking::KalmanLinear; tracking::FilterGoal m_filterGoal = tracking::FilterCenter; tracking::LostTrackType m_lostTrackType = tracking::TrackKCF; tracking::MatchType m_matchType = tracking::MatchHungrian; + std::array m_distType; + /// /// \brief m_dt /// Time step for Kalman @@ -38,7 +41,7 @@ struct TrackerSettings /// \brief m_distThres /// Distance threshold for Assignment problem for tracking::DistCenters or for tracking::DistRects (for tracking::DistJaccard it need from 0 to 1) /// - track_t m_distThres = 50; + track_t m_distThres = 0.5f; /// /// \brief m_maximumAllowedSkippedFrames @@ -68,6 +71,40 @@ struct TrackerSettings /// After this time (in seconds) the abandoned object will be removed /// int m_maxStaticTime = 25; + + /// + TrackerSettings() + { + m_distType[tracking::DistCenters] = 0.0f; + m_distType[tracking::DistRects] = 0.0f; + m_distType[tracking::DistJaccard] = 0.5f; + m_distType[tracking::DistHist] = 0.5f; + m_distType[tracking::DistHOG] = 0.0f; + + assert(CheckDistance()); + } + + /// + bool CheckDistance() const + { + track_t sum = std::accumulate(m_distType.begin(), m_distType.end(), 0.0f); + track_t maxOne = std::max(1.0f, std::fabs(sum)); + return std::fabs(sum - 1.0f) <= std::numeric_limits::epsilon() * maxOne; + } + + /// + bool SetDistances(std::array distType) + { + bool res = true; + auto oldDists = m_distType; + m_distType = distType; + if (!CheckDistance()) + { + m_distType = oldDists; + res = false; + } + return res; + } }; /// @@ -77,9 +114,14 @@ class CTracker { public: CTracker(const TrackerSettings& settings); + CTracker(const CTracker&) = delete; + CTracker(CTracker&&) = delete; + CTracker& operator=(const CTracker&) = delete; + CTracker& operator=(CTracker&&) = delete; + ~CTracker(void); - void Update(const regions_t& regions, cv::UMat grayFrame, float fps); + void Update(const regions_t& regions, cv::UMat currFrame, float fps); /// /// \brief GrayFrameToTrack @@ -129,10 +171,10 @@ class CTracker cv::UMat m_prevFrame; - void CreateDistaceMatrix(const regions_t& regions, distMatrix_t& costMatrix, track_t maxPossibleCost, track_t& maxCost); + void CreateDistaceMatrix(const regions_t& regions, distMatrix_t& costMatrix, track_t maxPossibleCost, track_t& maxCost, cv::UMat currFrame); void SolveHungrian(const distMatrix_t& costMatrix, size_t N, size_t M, assignments_t& assignment); void SolveBipartiteGraphs(const distMatrix_t& costMatrix, size_t N, size_t M, assignments_t& assignment, track_t maxCost); - void UpdateTrackingState(const regions_t& regions, cv::UMat grayFrame, float fps); + void UpdateTrackingState(const regions_t& regions, cv::UMat currFrame, float fps); }; diff --git a/src/Tracker/track.cpp b/src/Tracker/track.cpp index ef5bc20e3..c6db113ba 100644 --- a/src/Tracker/track.cpp +++ b/src/Tracker/track.cpp @@ -47,22 +47,22 @@ CTrack::CTrack( } /// -/// \brief CalcDist -/// \param pt +/// \brief CTrack::CalcDistCenter +/// \param reg /// \return /// -track_t CTrack::CalcDist(const Point_t& pt) const +track_t CTrack::CalcDistCenter(const CRegion& reg) const { - Point_t diff = m_predictionPoint - pt; + Point_t diff = m_predictionPoint - reg.m_rrect.center; return sqrtf(sqr(diff.x) + sqr(diff.y)); } /// -/// \brief CTrack::CalcDist +/// \brief CTrack::CalcDistRect /// \param reg /// \return /// -track_t CTrack::CalcDist(const CRegion& reg) const +track_t CTrack::CalcDistRect(const CRegion& reg) const { std::array diff; diff[0] = reg.m_rrect.center.x - m_lastRegion.m_rrect.center.x; @@ -92,6 +92,32 @@ track_t CTrack::CalcDistJaccard(const CRegion& reg) const return 1 - intArea / unionArea; } +/// +/// \brief CTrack::CalcDistHist +/// \param reg +/// \return +/// +track_t CTrack::CalcDistHist(const CRegion& reg, cv::UMat currFrame) const +{ + track_t intArea = static_cast((reg.m_brect & m_lastRegion.m_brect).area()); + track_t unionArea = static_cast(reg.m_brect.area() + m_lastRegion.m_brect.area() - intArea); + + return 1 - intArea / unionArea; +} + +/// +/// \brief CTrack::CalcDistHOG +/// \param reg +/// \return +/// +track_t CTrack::CalcDistHOG(const CRegion& reg) const +{ + track_t intArea = static_cast((reg.m_brect & m_lastRegion.m_brect).area()); + track_t unionArea = static_cast(reg.m_brect.area() + m_lastRegion.m_brect.area() - intArea); + + return 1 - intArea / unionArea; +} + /// /// \brief CTrack::CheckType /// \param type @@ -380,7 +406,7 @@ void CTrack::RectUpdate( bool inited = false; if (!m_tracker || m_tracker.empty()) { - CreateExternalTracker(); + CreateExternalTracker(currFrame.channels()); cv::Rect2d lastRect(brect.x - roiRect.x, brect.y - roiRect.y, brect.width, brect.height); if (m_staticFrame.empty()) @@ -464,7 +490,7 @@ void CTrack::RectUpdate( cv::Rect brect = m_predictionRect.boundingRect(); if (!m_VOTTracker) { - CreateExternalTracker(); + CreateExternalTracker(currFrame.channels()); cv::Rect2d lastRect(brect.x, brect.y, brect.width, brect.height); if (!m_staticFrame.empty()) @@ -553,7 +579,7 @@ void CTrack::RectUpdate( /// /// \brief CreateExternalTracker /// -void CTrack::CreateExternalTracker() +void CTrack::CreateExternalTracker(int channels) { switch (m_externalTrackerForLost) { @@ -575,9 +601,18 @@ void CTrack::CreateExternalTracker() if (!m_tracker || m_tracker.empty()) { cv::TrackerKCF::Params params; - params.compressed_size = 1; - params.desc_pca = cv::TrackerKCF::GRAY; - params.desc_npca = cv::TrackerKCF::GRAY; + if (channels == 1) + { + params.compressed_size = 1; + params.desc_pca = cv::TrackerKCF::GRAY; + params.desc_npca = cv::TrackerKCF::GRAY; + } + else + { + params.compressed_size = 3; + params.desc_pca = cv::TrackerKCF::CN; + params.desc_npca = cv::TrackerKCF::CN; + } params.resize = true; params.detect_thresh = 0.7f; #if (((CV_VERSION_MAJOR == 3) && (CV_VERSION_MINOR >= 3)) || (CV_VERSION_MAJOR > 3)) diff --git a/src/Tracker/track.h b/src/Tracker/track.h index 8508c4441..4d18ac5ba 100644 --- a/src/Tracker/track.h +++ b/src/Tracker/track.h @@ -240,24 +240,39 @@ class CTrack /// /// \brief CalcDist /// Euclidean distance in pixels between objects centres on two N and N+1 frames - /// \param pt + /// \param reg /// \return /// - track_t CalcDist(const Point_t& pt) const; + track_t CalcDistCenter(const CRegion& reg) const; /// /// \brief CalcDist /// Euclidean distance in pixels between object contours on two N and N+1 frames - /// \param r + /// \param reg /// \return /// - track_t CalcDist(const CRegion& reg) const; + track_t CalcDistRect(const CRegion& reg) const; /// /// \brief CalcDistJaccard /// Jaccard distance from 0 to 1 between object bounding rectangles on two N and N+1 frames - /// \param r + /// \param reg /// \return /// track_t CalcDistJaccard(const CRegion& reg) const; + /// + /// \brief CalcDistJaccard + /// Distance from 0 to 1 between objects histogramms on two N and N+1 frames + /// \param reg + /// \param currFrame + /// \return + /// + track_t CalcDistHist(const CRegion& reg, cv::UMat currFrame) const; + /// + /// \brief CalcDistHOG + /// Euclidean distance from 0 to 1 between HOG descriptors on two N and N+1 frames + /// \param reg + /// \return + /// + track_t CalcDistHOG(const CRegion& reg) const; bool CheckType(const std::string& type) const; @@ -297,7 +312,7 @@ class CTrack void RectUpdate(const CRegion& region, bool dataCorrect, cv::UMat prevFrame, cv::UMat currFrame); - void CreateExternalTracker(); + void CreateExternalTracker(int channels); void PointUpdate(const Point_t& pt, const cv::Size& newObjSize, bool dataCorrect, const cv::Size& frameSize); diff --git a/src/defines.h b/src/defines.h index f50f112bc..68d01c32d 100644 --- a/src/defines.h +++ b/src/defines.h @@ -106,9 +106,12 @@ enum Detectors /// enum DistType { - DistCenters = 0, - DistRects = 1, - DistJaccard = 2 + DistCenters, // Euclidean distance between centers, pixels + DistRects, // Euclidean distance between bounding rectangles, pixels + DistJaccard, // Intersection over Union, IoU, [0, 1] + DistHist, // Bhatacharia distance between histograms, [0, 1] + DistHOG, // Euclidean distance between HOG descriptors, [0, 1] + DistsCount }; /// @@ -116,8 +119,8 @@ enum DistType /// enum FilterGoal { - FilterCenter = 0, - FilterRect = 1 + FilterCenter, + FilterRect }; /// @@ -125,8 +128,8 @@ enum FilterGoal /// enum KalmanType { - KalmanLinear = 0, - KalmanUnscented = 1, + KalmanLinear, + KalmanUnscented, KalmanAugmentedUnscented }; @@ -135,8 +138,8 @@ enum KalmanType /// enum MatchType { - MatchHungrian = 0, - MatchBipart = 1 + MatchHungrian, + MatchBipart }; /// @@ -144,8 +147,8 @@ enum MatchType /// enum LostTrackType { - TrackNone = 0, - TrackKCF = 1, + TrackNone, + TrackKCF, TrackMIL, TrackMedianFlow, TrackGOTURN, From fcac65af9f8371aac44d2f9a8192b28bc1727232 Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Wed, 28 Aug 2019 23:41:43 +0300 Subject: [PATCH 3/8] Update TODO --- TODO | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/TODO b/TODO index 20092d0ef..d0f06657e 100644 --- a/TODO +++ b/TODO @@ -10,3 +10,15 @@ Deep SORT: 2. https://github.com/bitzy/DeepSort 3. https://github.com/oylz/DS +BGFG: +1. Original Vibe +2. Integration with new https://github.com/andrewssobral/bgslibrary + +Metrics between regions and tracking objects: +1. Histograms +2. HOG +3. Caching results of hist and HOG +4. Complex metric + +New runtime: +1. Integration with deepstream sdk 4.0 From fef8385465150b29bb76771a583cb531cfbd065f Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Thu, 29 Aug 2019 23:04:31 +0300 Subject: [PATCH 4/8] Color tracking by default --- cars_counting/CarsCounting.cpp | 2 +- example/VideoExample.cpp | 2 +- src/Tracker/Ctracker.h | 13 +++++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cars_counting/CarsCounting.cpp b/cars_counting/CarsCounting.cpp index 28d84cf43..d87652c13 100644 --- a/cars_counting/CarsCounting.cpp +++ b/cars_counting/CarsCounting.cpp @@ -107,7 +107,7 @@ void CarsCounting::Process() int64 t1 = cv::getTickCount(); cv::UMat uframe; - if (!m_detector->CanGrayProcessing() || !m_tracker->GrayFrameToTrack()) + if (!m_detector->CanGrayProcessing() || m_tracker->CanColorFrameToTrack()) { uframe = colorFrame.getUMat(cv::ACCESS_READ); } diff --git a/example/VideoExample.cpp b/example/VideoExample.cpp index e563dea42..75461cd5b 100644 --- a/example/VideoExample.cpp +++ b/example/VideoExample.cpp @@ -355,7 +355,7 @@ void VideoExample::Detection(cv::Mat frame, regions_t& regions) void VideoExample::Tracking(cv::Mat frame, const regions_t& regions) { cv::UMat uframe; - if (!m_tracker->GrayFrameToTrack()) + if (m_tracker->CanColorFrameToTrack()) { uframe = frame.getUMat(cv::ACCESS_READ); } diff --git a/src/Tracker/Ctracker.h b/src/Tracker/Ctracker.h index e79f033e3..f1b88b674 100644 --- a/src/Tracker/Ctracker.h +++ b/src/Tracker/Ctracker.h @@ -124,10 +124,10 @@ class CTracker void Update(const regions_t& regions, cv::UMat currFrame, float fps); /// - /// \brief GrayFrameToTrack + /// \brief CanGrayFrameToTrack /// \return /// - bool GrayFrameToTrack() const + bool CanGrayFrameToTrack() const { bool needColor = (m_settings.m_lostTrackType == tracking::LostTrackType::TrackGOTURN) || (m_settings.m_lostTrackType == tracking::LostTrackType::TrackDAT) || @@ -136,6 +136,15 @@ class CTracker return !needColor; } + /// + /// \brief CanColorFrameToTrack + /// \return + /// + bool CanColorFrameToTrack() const + { + return true; + } + /// /// \brief GetTracksCount /// \return From e10191cae7eccc69013ea994be3ddb7288d2a49b Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Fri, 30 Aug 2019 00:11:07 +0300 Subject: [PATCH 5/8] Added complex metric between objects with histograms comparing --- README.md | 20 ++++++++++---------- TODO | 5 +---- example/examples.h | 6 +++--- src/Tracker/track.cpp | 38 +++++++++++++++++++++++++++++--------- src/defines.h | 2 ++ 5 files changed, 45 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3910f87d2..98592d624 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Multitarget (multiple objects) tracker -#### 1. Objects detector can be created with function [CreateDetector](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Detector/BaseDetector.cpp#L17) with different values of the detectorType: +#### 1. Objects detector can be created with function [CreateDetector](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Detector/BaseDetector.cpp) with different values of the detectorType: 1.1. Based on background substraction: built-in Vibe (tracking::Motion_VIBE), SuBSENSE (tracking::Motion_SuBSENSE) and LOBSTER (tracking::Motion_LOBSTER); MOG2 (tracking::Motion_MOG2) from [opencv](https://github.com/opencv/opencv/blob/master/modules/video/include/opencv2/video/background_segm.hpp); MOG (tracking::Motion_MOG), GMG (tracking::Motion_GMG) and CNT (tracking::Motion_CNT) from [opencv_contrib](https://github.com/opencv/opencv_contrib/tree/master/modules/bgsegm). For foreground segmentation used contours from OpenCV with result as cv::RotatedRect @@ -18,27 +18,27 @@ 1.7. You can to use custom detector with bounding or rotated rectangle as output. -#### 2. Matching or solve an [assignment problem](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h#L23): +#### 2. Matching or solve an [assignment problem](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h): 2.1. Hungrian algorithm (tracking::MatchHungrian) with cubic time O(N^3) where N is objects count 2.2. Algorithm based on weighted bipartite graphs (tracking::MatchBipart) from [rdmpage](https://github.com/rdmpage/maximum-weighted-bipartite-matching) with time O(M * N^2) where N is objects count and M is connections count between detections on frame and tracking objects. It can be faster than Hungrian algorithm -2.3. [Distance](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h#L19) from detections and objects: euclidean distance in pixels between centers (tracking::DistCenters), euclidean distance in pixels between rectangles (tracking::DistRects), Jaccard or IoU distance from 0 to 1 (tracking::DistJaccard) +2.3. [Distance](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h) from detections and objects: euclidean distance in pixels between centers (tracking::DistCenters), euclidean distance in pixels between rectangles (tracking::DistRects), Jaccard or IoU distance from 0 to 1 (tracking::DistJaccard) -#### 3. [Smoothing trajectories and predict missed objects](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h#L20): +#### 3. [Smoothing trajectories and predict missed objects](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h): 3.1. Linear Kalman filter from OpenCV (tracking::KalmanLinear) 3.2. Unscented Kalman filter from OpenCV (tracking::KalmanUnscented) -3.3. [Kalman goal](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h#L21) is only coordinates (tracking::FilterCenter) or coordinates and size (tracking::FilterRect) +3.3. [Kalman goal](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h) is only coordinates (tracking::FilterCenter) or coordinates and size (tracking::FilterRect) -3.4. Simple [Abandoned detector](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h#L59) +3.4. Simple [Abandoned detector](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h) -3.5. [Line intersection](https://github.com/Smorodov/Multitarget-tracker/blob/master/cars_counting/CarsCounting.cpp#L381) counting +3.5. [Line intersection](https://github.com/Smorodov/Multitarget-tracker/blob/master/cars_counting/CarsCounting.cpp) counting -#### 4. [Advanced visual search](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h#L22) for objects if they have not been detected: +#### 4. [Advanced visual search](https://github.com/Smorodov/Multitarget-tracker/blob/master/src/Tracker/Ctracker.h) for objects if they have not been detected: 4.1. No search (tracking::TrackNone) @@ -48,7 +48,7 @@ With this option the tracking can work match slower but more accuracy. #### 5. Pipeline -5.1. Syncronous [pipeline - SyncProcess](https://github.com/Smorodov/Multitarget-tracker/blob/master/example/VideoExample.h#L77): +5.1. Syncronous [pipeline - SyncProcess](https://github.com/Smorodov/Multitarget-tracker/blob/master/example/VideoExample.h): - get frame from capture device; - decoding; - objects detection (1); @@ -57,7 +57,7 @@ With this option the tracking can work match slower but more accuracy. This pipeline is good if all algorithms are fast and works faster than time between two frames (40 ms for device with 25 fps). Or it can be used if we have only 1 core for all (no parallelization). -5.2. Pipeline with [2 threads - AsyncProcess](https://github.com/Smorodov/Multitarget-tracker/blob/master/example/VideoExample.h#L77): +5.2. Pipeline with [2 threads - AsyncProcess](https://github.com/Smorodov/Multitarget-tracker/blob/master/example/VideoExample.h): - 1th thread takes frame t and makes capture, decoding and objects detection; - 2th thread takes frame t-1, results from first thread and makes tracking and results presentation (this is the Main read). diff --git a/TODO b/TODO index d0f06657e..01de68d75 100644 --- a/TODO +++ b/TODO @@ -15,10 +15,7 @@ BGFG: 2. Integration with new https://github.com/andrewssobral/bgslibrary Metrics between regions and tracking objects: -1. Histograms -2. HOG -3. Caching results of hist and HOG -4. Complex metric +1. HOG New runtime: 1. Integration with deepstream sdk 4.0 diff --git a/example/examples.h b/example/examples.h index e0b5f29ae..0c995fc5f 100644 --- a/example/examples.h +++ b/example/examples.h @@ -60,14 +60,14 @@ class MotionDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistances({ 0.0f, 0.0f, 0.5f, 0.5f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; - settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving + settings.m_lostTrackType = tracking::TrackNone; // Use visual objects tracker for collisions resolving settings.m_matchType = tracking::MatchHungrian; settings.m_dt = 0.4f; // Delta time for Kalman filter settings.m_accelNoiseMag = 0.5f; // Accel noise magnitude for Kalman filter - settings.m_distThres = frame.rows / 10.f; // Distance threshold between region and object on two frames + settings.m_distThres = 0.6; // frame.rows / 10.f; // Distance threshold between region and object on two frames settings.m_useAbandonedDetection = false; if (settings.m_useAbandonedDetection) diff --git a/src/Tracker/track.cpp b/src/Tracker/track.cpp index c6db113ba..cc27febac 100644 --- a/src/Tracker/track.cpp +++ b/src/Tracker/track.cpp @@ -99,10 +99,33 @@ track_t CTrack::CalcDistJaccard(const CRegion& reg) const /// track_t CTrack::CalcDistHist(const CRegion& reg, cv::UMat currFrame) const { - track_t intArea = static_cast((reg.m_brect & m_lastRegion.m_brect).area()); - track_t unionArea = static_cast(reg.m_brect.area() + m_lastRegion.m_brect.area() - intArea); - - return 1 - intArea / unionArea; + track_t res = 1; + + if (reg.m_hist.empty()) + { + int bins = 64; + std::vector histSize; + std::vector ranges; + std::vector channels; + + for (int i = 0, stop = currFrame.channels(); i < stop; ++i) + { + histSize.push_back(bins); + ranges.push_back(0); + ranges.push_back(255); + channels.push_back(i); + } + + std::vector regROI = { currFrame(reg.m_brect) }; + cv::calcHist(regROI, channels, cv::Mat(), reg.m_hist, histSize, ranges, false); + cv::normalize(reg.m_hist, reg.m_hist, 0, 1, cv::NORM_MINMAX, -1, cv::Mat()); + } + if (!reg.m_hist.empty() && !m_lastRegion.m_hist.empty()) + { + res = static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_BHATTACHARYYA)); + } + + return res; } /// @@ -112,10 +135,7 @@ track_t CTrack::CalcDistHist(const CRegion& reg, cv::UMat currFrame) const /// track_t CTrack::CalcDistHOG(const CRegion& reg) const { - track_t intArea = static_cast((reg.m_brect & m_lastRegion.m_brect).area()); - track_t unionArea = static_cast(reg.m_brect.area() + m_lastRegion.m_brect.area() - intArea); - - return 1 - intArea / unionArea; + return 1; } /// @@ -130,7 +150,7 @@ bool CTrack::CheckType(const std::string& type) const /// /// \brief CTrack::Update -/// \param region +/// \*param region /// \param dataCorrect /// \param max_trace_length /// \param prevFrame diff --git a/src/defines.h b/src/defines.h index 68d01c32d..a04f64aa8 100644 --- a/src/defines.h +++ b/src/defines.h @@ -53,6 +53,8 @@ class CRegion std::string m_type; float m_confidence = -1; + mutable cv::Mat m_hist; + private: /// /// \brief R2BRect From 18e0d6ac8b549d934af0bb3c70b52c7e3b81f0d2 Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Fri, 30 Aug 2019 00:17:12 +0300 Subject: [PATCH 6/8] Default motion detector example with KCF and IoU --- TODO | 8 +++++++- example/examples.h | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/TODO b/TODO index 01de68d75..1efb90668 100644 --- a/TODO +++ b/TODO @@ -15,7 +15,13 @@ BGFG: 2. Integration with new https://github.com/andrewssobral/bgslibrary Metrics between regions and tracking objects: -1. HOG +1. Profiling histograms +2. HOG +3. Normaliza pixels metric to [0, 1] New runtime: 1. Integration with deepstream sdk 4.0 + +Tests: +1. Quality tests +2. Performance tests \ No newline at end of file diff --git a/example/examples.h b/example/examples.h index 0c995fc5f..61b40e4f8 100644 --- a/example/examples.h +++ b/example/examples.h @@ -60,14 +60,14 @@ class MotionDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 0.0f, 0.5f, 0.5f, 0.0f }); + settings.SetDistances({ 0.0f, 0.0f, 1.0f, 0.0f, 0.0f }); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; - settings.m_lostTrackType = tracking::TrackNone; // Use visual objects tracker for collisions resolving + settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving settings.m_matchType = tracking::MatchHungrian; settings.m_dt = 0.4f; // Delta time for Kalman filter settings.m_accelNoiseMag = 0.5f; // Accel noise magnitude for Kalman filter - settings.m_distThres = 0.6; // frame.rows / 10.f; // Distance threshold between region and object on two frames + settings.m_distThres = 0.6f; // frame.rows / 10.f; // Distance threshold between region and object on two frames settings.m_useAbandonedDetection = false; if (settings.m_useAbandonedDetection) From 37c6826ff63eb7c2c40363c84493d3ffb10756c3 Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Fri, 6 Sep 2019 12:31:08 +0300 Subject: [PATCH 7/8] Fixed openmp types --- src/Tracker/Ctracker.cpp | 10 ++++++---- src/Tracker/track.cpp | 35 ++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/Tracker/Ctracker.cpp b/src/Tracker/Ctracker.cpp index 2780b1b34..a2dca48c2 100644 --- a/src/Tracker/Ctracker.cpp +++ b/src/Tracker/Ctracker.cpp @@ -123,9 +123,9 @@ void CTracker::UpdateTrackingState( } // Update Kalman Filters state - const ptrdiff_t stop_i = static_cast(assignment.size()); + const ptrdiff_t stop_i = static_cast(assignment.size()); #pragma omp parallel for - for (int i = 0; i < stop_i; ++i) + for (ptrdiff_t i = 0; i < stop_i; ++i) { // If track updated less than one time, than filter state is not correct. if (assignment[i] != -1) // If we have assigned detect, then update using its coordinates, @@ -156,11 +156,13 @@ void CTracker::CreateDistaceMatrix(const regions_t& regions, distMatrix_t& costM const size_t N = m_tracks.size(); // Tracking objects maxCost = 0; - for (size_t i = 0; i < m_tracks.size(); i++) + const ptrdiff_t stop_i = static_cast(m_tracks.size()); +//#pragma omp parallel for + for (ptrdiff_t i = 0; i < stop_i; ++i) { const auto& track = m_tracks[i]; - for (size_t j = 0; j < regions.size(); j++) + for (size_t j = 0; j < regions.size(); ++j) { auto dist = maxPossibleCost; if (m_tracks[i]->CheckType(regions[j].m_type)) diff --git a/src/Tracker/track.cpp b/src/Tracker/track.cpp index cc27febac..3f4c5f625 100644 --- a/src/Tracker/track.cpp +++ b/src/Tracker/track.cpp @@ -101,28 +101,29 @@ track_t CTrack::CalcDistHist(const CRegion& reg, cv::UMat currFrame) const { track_t res = 1; - if (reg.m_hist.empty()) - { - int bins = 64; - std::vector histSize; - std::vector ranges; - std::vector channels; - - for (int i = 0, stop = currFrame.channels(); i < stop; ++i) - { - histSize.push_back(bins); - ranges.push_back(0); - ranges.push_back(255); - channels.push_back(i); - } - - std::vector regROI = { currFrame(reg.m_brect) }; - cv::calcHist(regROI, channels, cv::Mat(), reg.m_hist, histSize, ranges, false); + if (reg.m_hist.empty()) + { + int bins = 64; + std::vector histSize; + std::vector ranges; + std::vector channels; + + for (int i = 0, stop = currFrame.channels(); i < stop; ++i) + { + histSize.push_back(bins); + ranges.push_back(0); + ranges.push_back(255); + channels.push_back(i); + } + + std::vector regROI = { currFrame(reg.m_brect) }; + cv::calcHist(regROI, channels, cv::Mat(), reg.m_hist, histSize, ranges, false); cv::normalize(reg.m_hist, reg.m_hist, 0, 1, cv::NORM_MINMAX, -1, cv::Mat()); } if (!reg.m_hist.empty() && !m_lastRegion.m_hist.empty()) { res = static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_BHATTACHARYYA)); + //res = 1.f - static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_CORREL)); } return res; From 151d513f51fa1852234977099fd9f784a254943f Mon Sep 17 00:00:00 2001 From: Nuzhny007 Date: Sun, 8 Sep 2019 07:19:52 +0300 Subject: [PATCH 8/8] Add simple method for one distance type --- async_detector/AsyncDetector.cpp | 2 +- cars_counting/CarsCounting.cpp | 2 +- example/MouseExample.h | 2 +- example/examples.h | 14 +++++++------- src/Tracker/Ctracker.h | 8 ++++++++ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/async_detector/AsyncDetector.cpp b/async_detector/AsyncDetector.cpp index dee3b50c8..a3367ffeb 100644 --- a/async_detector/AsyncDetector.cpp +++ b/async_detector/AsyncDetector.cpp @@ -298,7 +298,7 @@ void AsyncDetector::CaptureThread(std::string fileName, int startFrame, float* f const int minStaticTime = 5; TrackerSettings trackerSettings; - trackerSettings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); + trackerSettings.SetDistance(tracking::DistCenters); trackerSettings.m_kalmanType = tracking::KalmanLinear; trackerSettings.m_filterGoal = tracking::FilterRect; trackerSettings.m_lostTrackType = tracking::TrackKCF; // Use KCF tracker for collisions resolving diff --git a/cars_counting/CarsCounting.cpp b/cars_counting/CarsCounting.cpp index d87652c13..29726d4d4 100644 --- a/cars_counting/CarsCounting.cpp +++ b/cars_counting/CarsCounting.cpp @@ -259,7 +259,7 @@ bool CarsCounting::InitTracker(cv::UMat frame) m_detector->SetMinObjectSize(cv::Size(m_minObjWidth, m_minObjWidth)); TrackerSettings settings; - settings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistCenters); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use KCF tracker for collisions resolving diff --git a/example/MouseExample.h b/example/MouseExample.h index e2eab7c92..5e8712719 100644 --- a/example/MouseExample.h +++ b/example/MouseExample.h @@ -47,7 +47,7 @@ void MouseTracking(cv::CommandLineParser parser) cv::setMouseCallback("Video", mv_MouseCallback, (void*)&pointXY); TrackerSettings settings; - settings.SetDistances({ 1.0f, 0.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistCenters); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterCenter; settings.m_lostTrackType = tracking::TrackNone; diff --git a/example/examples.h b/example/examples.h index 61b40e4f8..0e9ed3249 100644 --- a/example/examples.h +++ b/example/examples.h @@ -60,14 +60,14 @@ class MotionDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 0.0f, 1.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistCenters); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving settings.m_matchType = tracking::MatchHungrian; settings.m_dt = 0.4f; // Delta time for Kalman filter settings.m_accelNoiseMag = 0.5f; // Accel noise magnitude for Kalman filter - settings.m_distThres = 0.6f; // frame.rows / 10.f; // Distance threshold between region and object on two frames + settings.m_distThres = frame.rows / 10.f; // Distance threshold between region and object on two frames settings.m_useAbandonedDetection = false; if (settings.m_useAbandonedDetection) @@ -175,7 +175,7 @@ class FaceDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 0.0f, 1.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistJaccard); settings.m_kalmanType = tracking::KalmanUnscented; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -271,7 +271,7 @@ class PedestrianDetectorExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistRects); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -368,7 +368,7 @@ class SSDMobileNetExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistRects); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -492,7 +492,7 @@ class YoloExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistRects); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackCSRT; // Use visual objects tracker for collisions resolving @@ -616,7 +616,7 @@ class YoloDarknetExample : public VideoExample bool InitTracker(cv::UMat frame) { TrackerSettings settings; - settings.SetDistances({ 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }); + settings.SetDistance(tracking::DistRects); settings.m_kalmanType = tracking::KalmanLinear; settings.m_filterGoal = tracking::FilterRect; settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving diff --git a/src/Tracker/Ctracker.h b/src/Tracker/Ctracker.h index f1b88b674..0a679a46f 100644 --- a/src/Tracker/Ctracker.h +++ b/src/Tracker/Ctracker.h @@ -105,6 +105,14 @@ struct TrackerSettings } return res; } + + /// + bool SetDistance(tracking::DistType distType) + { + std::fill(m_distType.begin(), m_distType.end(), 0.0f); + m_distType[distType] = 1.f; + return true; + } }; ///