From 29bc98331f8034bc290ee83c0d61438251bd8226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Tue, 18 Feb 2020 12:19:26 +0100 Subject: [PATCH 1/8] Add Example no7 for SmartCamOperator, Fix Compile-Errors, Minor Changes TODO: Add Ring Memory, Crop and Zoom + Smoothing --- CMakeLists.txt | 4 +- example/examples.h | 155 +++++++++++++++++++++++++++ example/main.cpp | 7 ++ src/Detector/YoloDarknetDetector.cpp | 1 + src/Tracker/track.cpp | 2 +- 5 files changed, 166 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f30454e8d..5bc3e5f03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,12 +45,12 @@ if (BUILD_CARS_COUNTING) add_subdirectory(cars_counting) endif(BUILD_CARS_COUNTING) -option(BUILD_ASYNC_DETECTOR "Should compiled async example with low fps Detector?" OFF) +option(BUILD_ASYNC_DETECTOR "Should compiled async example with low fps Detector?" ON) if (BUILD_ASYNC_DETECTOR) add_subdirectory(async_detector) endif(BUILD_ASYNC_DETECTOR) -option(BUILD_YOLO_LIB "Should compiled standalone yolo_lib with original darknet?" OFF) +option(BUILD_YOLO_LIB "Should compiled standalone yolo_lib with original darknet?" ON) if (BUILD_YOLO_LIB) add_subdirectory(src/Detector/darknet) add_definitions(-DBUILD_YOLO_LIB) diff --git a/example/examples.h b/example/examples.h index ad881dd09..74ed567da 100644 --- a/example/examples.h +++ b/example/examples.h @@ -727,3 +727,158 @@ class YoloDarknetExample : public VideoExample }; #endif + + +// ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // +// -------------- Smart Camera Operator Example Settings ---------------- // +// ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // + + +#ifdef BUILD_YOLO_LIB +// ---------------------------------------------------------------------- + +/// +/// \brief The SmartCamOperator_w_YoloDarknet class +/// +class SmartCamOperator_w_YoloDarknet : public VideoExample +{ +public: + SmartCamOperator_w_YoloDarknet(const cv::CommandLineParser& parser) + : + VideoExample(parser) + { + } + +protected: + /// + /// \brief InitDetector + /// \param frame + /// \return + /// + bool InitDetector(cv::UMat frame) + { + config_t config; + +#ifdef _WIN32 + std::string pathToModel = "../../data/"; +#else + std::string pathToModel = "../data/"; +#endif +#if 0 + config.emplace("modelConfiguration", pathToModel + "yolov3-tiny.cfg"); + config.emplace("modelBinary", pathToModel + "yolov3-tiny.weights"); + config.emplace("confidenceThreshold", "0.5"); +#else + config.emplace("modelConfiguration", pathToModel + "yolov3.cfg"); + config.emplace("modelBinary", pathToModel + "yolov3.weights"); + config.emplace("confidenceThreshold", "0.6"); +#endif + config.emplace("classNames", pathToModel + "horse_and_rider.names"); + config.emplace("maxCropRatio", "-1"); + + config.emplace("white_list", "Horse&Rider"); + + m_detector = std::unique_ptr(CreateDetector(tracking::Detectors::Yolo_Darknet, config, frame)); + if (m_detector.get()) + { + m_detector->SetMinObjectSize(cv::Size(frame.cols / 40, frame.rows / 40)); + return true; + } + return false; + } + + /// + /// \brief InitTracker + /// \param frame + /// \return + /// + bool InitTracker(cv::UMat frame) + { + TrackerSettings settings; + settings.SetDistance(tracking::DistCenters); + settings.m_kalmanType = tracking::KalmanLinear; + settings.m_filterGoal = tracking::FilterCenter; + settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving + settings.m_matchType = tracking::MatchHungrian; + settings.m_dt = 0.3f; // Delta time for Kalman filter + settings.m_accelNoiseMag = 0.2f; // Accel noise magnitude for Kalman filter + settings.m_distThres = 0.8f; // Distance threshold between region and object on two frames + settings.m_minAreaRadius = frame.rows / 20.f; + settings.m_maximumAllowedSkippedFrames = cvRound(m_fps); // Maximum allowed skipped frames + settings.m_maxTraceLength = cvRound(2 * m_fps); // Maximum trace length + + /* + settings.AddNearTypes("car", "bus", false); + settings.AddNearTypes("car", "truck", false); + settings.AddNearTypes("person", "bicycle", true); + settings.AddNearTypes("person", "motorbike", true); + */ + + m_tracker = std::make_unique(settings); + + return true; + } + + /// + /// \brief DrawData + /// \param frame + /// \param framesCounter + /// \param currTime + /// + void DrawData(cv::Mat frame, int framesCounter, int currTime) + { + auto tracks = m_tracker->GetTracks(); + + if (m_showLogs) + { + std::cout << "Frame " << framesCounter << ": tracks = " << tracks.size() << ", time = " << currTime << std::endl; + } + + for (const auto& track : tracks) + { + if (track.IsRobust(2, // Minimal trajectory size + 0.5f, // Minimal ratio raw_trajectory_points / trajectory_lenght + cv::Size2f(0.1f, 8.0f)) // Min and max ratio: width / height + ) + { + DrawTrack(frame, 1, track); + + + std::stringstream label; + label << track.m_type << " " << std::setprecision(2) << track.m_velocity << ": " << track.m_confidence; + int baseLine = 0; + cv::Size labelSize = cv::getTextSize(label.str(), cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + cv::Rect brect = track.m_rrect.boundingRect(); + if (brect.x < 0) + { + brect.width = std::min(brect.width, frame.cols - 1); + brect.x = 0; + } + else if (brect.x + brect.width >= frame.cols) + { + brect.x = std::max(0, frame.cols - brect.width - 1); + brect.width = std::min(brect.width, frame.cols - 1); + } + if (brect.y - labelSize.height < 0) + { + brect.height = std::min(brect.height, frame.rows - 1); + brect.y = labelSize.height; + } + else if (brect.y + brect.height >= frame.rows) + { + brect.y = std::max(0, frame.rows - brect.height - 1); + brect.height = std::min(brect.height, frame.rows - 1); + } + DrawFilledRect(frame, cv::Rect(cv::Point(brect.x, brect.y - labelSize.height), cv::Size(labelSize.width, labelSize.height + baseLine)), cv::Scalar(200, 200, 200), 150); + cv::putText(frame, label.str(), brect.tl(), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); + } + } + + //m_detector->CalcMotionMap(frame); + } +}; + +#endif diff --git a/example/main.cpp b/example/main.cpp index 2eed72439..9a04a4ce8 100644 --- a/example/main.cpp +++ b/example/main.cpp @@ -93,6 +93,13 @@ int main(int argc, char** argv) asyncPipeline ? yolo_detector.AsyncProcess() : yolo_detector.SyncProcess(); break; } + + case 7: + { + SmartCamOperator_w_YoloDarknet smart_cam_operator(parser); + asyncPipeline ? smart_cam_operator.AsyncProcess() : smart_cam_operator.SyncProcess(); + break; + } #endif default: diff --git a/src/Detector/YoloDarknetDetector.cpp b/src/Detector/YoloDarknetDetector.cpp index e1317e5cf..c607013f7 100644 --- a/src/Detector/YoloDarknetDetector.cpp +++ b/src/Detector/YoloDarknetDetector.cpp @@ -1,3 +1,4 @@ +#include #include "YoloDarknetDetector.h" #include "nms.h" diff --git a/src/Tracker/track.cpp b/src/Tracker/track.cpp index 1cacc7c4b..5ca6424bb 100644 --- a/src/Tracker/track.cpp +++ b/src/Tracker/track.cpp @@ -120,7 +120,7 @@ track_t CTrack::CalcDistHist(const CRegion& reg, cv::UMat currFrame) const } if (!reg.m_hist.empty() && !m_lastRegion.m_hist.empty()) { -#if (((CV_VERSION_MAJOR == 4) && (CV_VERSION_MINOR < 2)) || (CV_VERSION_MAJOR == 3)) +#if (((CV_VERSION_MAJOR == 4) && (CV_VERSION_MINOR < 1)) || (CV_VERSION_MAJOR == 3)) res = static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_BHATTACHARYYA)); //res = 1.f - static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_CORREL)); #else From 16d07180bb33f4db495a5e7d76ac98e67e79117f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Tue, 18 Feb 2020 12:33:26 +0100 Subject: [PATCH 2/8] Upgrade .gitignore to exclude HorseRider Data for SmartCamOperator --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index e3814ab6b..cee3bf18c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ build CMakeLists.txt.user* +# SmartCameraOperator for Horses and Riders +# Data, Weights, other stuff + +data/Kirchhellen*.avi +data/*.names +data/yolov3.cfg +data/yolov3.weights From 8022fb5899328e4f388b221f583df64217a2d3f0 Mon Sep 17 00:00:00 2001 From: Timmimim Date: Tue, 18 Feb 2020 17:11:16 +0100 Subject: [PATCH 3/8] Add first minor steps towards CropZoom --- example/examples.h | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/example/examples.h b/example/examples.h index 74ed567da..711f42bc4 100644 --- a/example/examples.h +++ b/example/examples.h @@ -6,6 +6,14 @@ #include "VideoExample.h" +struct bounding_box +{ + int x; + int y; + int w; + int h; +}; + /// /// \brief DrawFilledRect /// @@ -735,8 +743,9 @@ class YoloDarknetExample : public VideoExample // ---------------------------------------------------------------------- // // ---------------------------------------------------------------------- // +#define bb_history 50 -#ifdef BUILD_YOLO_LIB +//#ifdef BUILD_YOLO_LIB // ---------------------------------------------------------------------- /// @@ -749,9 +758,22 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample : VideoExample(parser) { + // Init RingMemory to show + cv::UMat frame = m_frameInfo[0].m_frame.getUMat(cv::ACCESS_READ); + for (int i = 0; i < bb_history; i++) + { + bb_ring_memory[i] = {0, 0, frame.cols, frame.rows}; + } } protected: + + /// + /// \brief Ring Memory + /// + struct bounding_box bb_ring_memory[bb_history]; + int ring_mem_index = 0; + /// /// \brief InitDetector /// \param frame @@ -879,6 +901,8 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample //m_detector->CalcMotionMap(frame); } + + }; #endif From c8e11fe312ee463adfac50da1cd22e9bf8e7e58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Wed, 19 Feb 2020 02:22:38 +0100 Subject: [PATCH 4/8] Update CMake file for C++17, std::filesystem --- CMakeLists.txt | 4 ++-- example/CMakeLists.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bc3e5f03..c1849f1b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,10 +12,10 @@ if (OPENMP_FOUND) list(APPEND CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) if (CMAKE_COMPILER_IS_GNUCXX) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic-errors -std=c++14" CACHE STRING COMPILE_FLAGS FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic-errors -std=c++17 -lstdc++fs" CACHE STRING COMPILE_FLAGS FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -march=native -mtune=native -funroll-loops -Wall -DNDEBUG -DBOOST_DISABLE_ASSERTS" CACHE STRING COMPILE_FLAGS FORCE) set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -march=native -mtune=native -Wall -DDEBUG" CACHE STRING COMPILE_FLAGS FORCE) elseif (MSVC) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b57f99e0a..c44b94d52 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -47,3 +47,4 @@ endif(BUILD_YOLO_LIB) ADD_EXECUTABLE(${PROJECT_NAME} ${SOURCES} ${HEADERS}) TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${LIBS}) +target_link_libraries(${PROJECT_NAME} /usr/lib/gcc/x86_64-linux-gnu/7/libstdc++fs.a) From bcb16930a4e0a561194436e5d1590656114f79f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Wed, 19 Feb 2020 02:25:42 +0100 Subject: [PATCH 5/8] Add RingMemory, Crop&Zoom, new Saver for ZoomVideo, and others Variable ring memory for past regions of interest. ROI join, scale to 16:9 aspect ratio, crop and resize for simulated zoom. New saver to write to an extra video file simultaneously, with name depending on original output video name. Multiple small additions and changes to accommodate SmartCamOperator. --- example/VideoExample.cpp | 48 +++++ example/VideoExample.h | 9 + example/examples.h | 444 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 490 insertions(+), 11 deletions(-) diff --git a/example/VideoExample.cpp b/example/VideoExample.cpp index de8dc32a1..2cace030e 100644 --- a/example/VideoExample.cpp +++ b/example/VideoExample.cpp @@ -2,6 +2,10 @@ #include #include +#include +#include +namespace fs = std::experimental::filesystem; + #include "VideoExample.h" /// @@ -23,6 +27,16 @@ VideoExample::VideoExample(const cv::CommandLineParser& parser) { m_inFile = parser.get(0); m_outFile = parser.get("out"); + + fs::path path(parser.get("out")); + auto filename = path.stem(); + filename += "_zoomed"; + auto ext = path.extension(); + filename += ext.string(); + auto parent = path.parent_path(); + path = parent / filename; + m_outFileZoomed = path.string(); + m_showLogs = parser.get("show_logs") != 0; m_startFrame = parser.get("start_frame"); m_endFrame = parser.get("end_frame"); @@ -45,6 +59,7 @@ VideoExample::VideoExample(const cv::CommandLineParser& parser) void VideoExample::SyncProcess() { cv::VideoWriter writer; + cv::VideoWriter writerZoom; #ifndef SILENT_WORK cv::namedWindow("Video", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO); @@ -109,6 +124,10 @@ void VideoExample::SyncProcess() allTime += t2 - t1; int currTime = cvRound(1000 * (t2 - t1) / freq); + // Crop, Zoom, and Save to new Video file + cv::Mat zoomedFrame = ZoomInOnROI(frame, framesCounter, currTime); + WriteZoomedFrame(writerZoom, zoomedFrame); + DrawData(frame, framesCounter, currTime); #ifndef SILENT_WORK @@ -156,6 +175,7 @@ void VideoExample::AsyncProcess() std::thread thCapDet(CaptureAndDetect, this, std::ref(stopCapture)); cv::VideoWriter writer; + cv::VideoWriter writerZoom; #ifndef SILENT_WORK cv::namedWindow("Video", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO); @@ -204,6 +224,10 @@ void VideoExample::AsyncProcess() //std::cout << "Frame " << framesCounter << ": td = " << (1000 * frameInfo.m_dt / freq) << ", tt = " << (1000 * (t2 - t1) / freq) << std::endl; + // Crop, Zoom, and Save to new Video file + cv::Mat zoomedFrame = ZoomInOnROI(frameInfo.m_frame, framesCounter, currTime); + WriteZoomedFrame(writerZoom, zoomedFrame); + DrawData(frameInfo.m_frame, framesCounter, currTime); WriteFrame(writer, frameInfo.m_frame); @@ -472,3 +496,27 @@ bool VideoExample::WriteFrame(cv::VideoWriter& writer, const cv::Mat& frame) } return false; } + +/// +/// \brief VideoExample::WriteFrame +/// \param writer +/// \param frame +/// \return +/// +bool VideoExample::WriteZoomedFrame(cv::VideoWriter& writer, const cv::Mat& frame) +{ + if (!m_outFile.empty()) + { + if (!writer.isOpened()) + { + + writer.open(m_outFileZoomed, m_fourcc, m_fps, frame.size(), true); + } + if (writer.isOpened()) + { + writer << frame; + return true; + } + } + return false; +} diff --git a/example/VideoExample.h b/example/VideoExample.h index 24ed7aaf1..214028bdb 100644 --- a/example/VideoExample.h +++ b/example/VideoExample.h @@ -29,6 +29,11 @@ class VideoExample void AsyncProcess(); void SyncProcess(); + cv::UMat getFrame() + { + return this->m_frameInfo[0].m_frame.getUMat(cv::ACCESS_READ); + } + protected: std::unique_ptr m_detector; std::unique_ptr m_tracker; @@ -49,6 +54,8 @@ class VideoExample virtual void DrawData(cv::Mat frame, int framesCounter, int currTime) = 0; + virtual cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) = 0; + void DrawTrack(cv::Mat frame, int resizeCoeff, const TrackingObject& track, bool drawTrajectory = true); private: @@ -56,6 +63,7 @@ class VideoExample bool m_isDetectorInitialized = false; std::string m_inFile; std::string m_outFile; + std::string m_outFileZoomed; int m_fourcc = cv::VideoWriter::fourcc('M', 'J', 'P', 'G'); int m_startFrame = 0; int m_endFrame = 0; @@ -76,4 +84,5 @@ class VideoExample bool OpenCapture(cv::VideoCapture& capture); bool WriteFrame(cv::VideoWriter& writer, const cv::Mat& frame); + bool WriteZoomedFrame(cv::VideoWriter& writer, const cv::Mat& frame); }; diff --git a/example/examples.h b/example/examples.h index 711f42bc4..441a9833c 100644 --- a/example/examples.h +++ b/example/examples.h @@ -1,9 +1,12 @@ #pragma once #include +#include #include #include +#include + #include "VideoExample.h" struct bounding_box @@ -162,6 +165,22 @@ class MotionDetectorExample : public VideoExample m_detector->CalcMotionMap(frame); } + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } private: int m_minObjWidth = 8; int m_minStaticTime = 5; @@ -258,6 +277,23 @@ class FaceDetectorExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; // ---------------------------------------------------------------------- @@ -355,6 +391,23 @@ class PedestrianDetectorExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; // ---------------------------------------------------------------------- @@ -461,6 +514,23 @@ class SSDMobileNetExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; // ---------------------------------------------------------------------- @@ -582,6 +652,23 @@ class YoloExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; #ifdef BUILD_YOLO_LIB @@ -732,6 +819,23 @@ class YoloDarknetExample : public VideoExample //m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; #endif @@ -743,9 +847,9 @@ class YoloDarknetExample : public VideoExample // ---------------------------------------------------------------------- // // ---------------------------------------------------------------------- // -#define bb_history 50 +#define bb_history 100 -//#ifdef BUILD_YOLO_LIB +#ifdef BUILD_YOLO_LIB // ---------------------------------------------------------------------- /// @@ -758,12 +862,7 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample : VideoExample(parser) { - // Init RingMemory to show - cv::UMat frame = m_frameInfo[0].m_frame.getUMat(cv::ACCESS_READ); - for (int i = 0; i < bb_history; i++) - { - bb_ring_memory[i] = {0, 0, frame.cols, frame.rows}; - } + } protected: @@ -772,7 +871,9 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample /// \brief Ring Memory /// struct bounding_box bb_ring_memory[bb_history]; - int ring_mem_index = 0; + + int frames_since_init = 0; + int frames_without_detected_objects = 0; /// /// \brief InitDetector @@ -828,7 +929,7 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample settings.m_accelNoiseMag = 0.2f; // Accel noise magnitude for Kalman filter settings.m_distThres = 0.8f; // Distance threshold between region and object on two frames settings.m_minAreaRadius = frame.rows / 20.f; - settings.m_maximumAllowedSkippedFrames = cvRound(m_fps); // Maximum allowed skipped frames + settings.m_maximumAllowedSkippedFrames = cvRound(.8 * m_fps); // Maximum allowed skipped frames settings.m_maxTraceLength = cvRound(2 * m_fps); // Maximum trace length /* @@ -840,6 +941,12 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample m_tracker = std::make_unique(settings); + // Init RingMemory to show + for (int i = 0; i < bb_history; i++) + { + bb_ring_memory[i] = {0, 0, frame.cols-1, frame.rows-1}; + } + return true; } @@ -867,7 +974,6 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample { DrawTrack(frame, 1, track); - std::stringstream label; label << track.m_type << " " << std::setprecision(2) << track.m_velocity << ": " << track.m_confidence; int baseLine = 0; @@ -903,6 +1009,322 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample } + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + auto tracks = m_tracker->GetTracks(); + + if (m_showLogs) + { + std::cout << "Frame " << framesCounter << ": tracks = " << tracks.size() << ", time = " << currTime << std::endl; + } + + if (tracks.size() == 0 && frames_without_detected_objects <= 100) + { + bb_ring_memory[framesCounter % bb_history] = bb_ring_memory[(framesCounter - 1) % bb_history]; + ++frames_without_detected_objects; + } + else if (tracks.size() == 0 && frames_without_detected_objects > 100) + { + struct bounding_box entire_frame {0, 0, frame.cols, frame.rows}; + bb_ring_memory[framesCounter % bb_history] = entire_frame; + } + else + { + frames_without_detected_objects = 0; + struct bounding_box joint_ROI {frame.cols, frame.rows, 0, 0}; + for (const auto& track : tracks) + { + cv::Rect brect = track.m_rrect.boundingRect(); + + if (brect.x < 0) + { + brect.width = std::min(brect.width, frame.cols - 1); + brect.x = 0; + } + else if ((brect.x + brect.width) >= frame.cols) + { + brect.x = std::max(0, frame.cols - brect.width - 1); + brect.width = std::min(brect.width, frame.cols - 1); + } + if (brect.y < 0) + { + brect.height = std::min(brect.height, frame.rows - 1); + brect.y = 0; + } + else if ((brect.y + brect.height) >= frame.rows) + { + brect.y = std::max(0, frame.rows - brect.height - 1); + brect.height = std::min(brect.height, frame.rows - 1); + } + joint_ROI.x = (brect.x < joint_ROI.x) ? brect.x : joint_ROI.x; + joint_ROI.y = (brect.y < joint_ROI.y) ? brect.y : joint_ROI.y; + joint_ROI.w = ((brect.x + brect.width) > joint_ROI.w) ? (brect.x + brect.width) : joint_ROI.w; + joint_ROI.h = ((brect.y + brect.height) > joint_ROI.h) ? (brect.y + brect.height) : joint_ROI.h; + + } + joint_ROI.w = joint_ROI.w - joint_ROI.x; + joint_ROI.h = joint_ROI.h - joint_ROI.y; + + struct bounding_box ROI_raw = joint_ROI; + + + if (ROI_raw.x - 50 >= 0) ROI_raw.x -= 50; + if (ROI_raw.y - 50 >= 0) ROI_raw.y -= 50; + if (ROI_raw.x + ROI_raw.w + 100 <= frame.cols) ROI_raw.w += 100; + if (ROI_raw.y + ROI_raw.h + 100 <= frame.rows) ROI_raw.h += 100; + + + struct bounding_box ROI; + + // check for and extablish 16:9 aspect ratio + // TODO: This probably needs testing and more case coverage! + if ((ROI_raw.w / ROI_raw.h) < 1.7777777) + { + ROI.y = ROI_raw.y; + ROI.w = (int) (ROI_raw.h * 1.7777777 + .5); + ROI.h = ROI_raw.h; + + if ((ROI_raw.x - .5 * (ROI.w - ROI_raw.w)) < 0) { + ROI.x = 0; + ROI.w -= (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); + } + else if ((ROI_raw.x + ROI_raw.w + .5 * (ROI.w - ROI_raw.w)) >= frame.cols) + { + ROI.x = (int) (ROI_raw.x - (frame.cols - 1 - (ROI_raw.x + .5 * (ROI.w - ROI_raw.w))) - .5); + ROI.w = frame.cols - 1; + } + else + { + ROI.x = (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); + } + } + else + if ((ROI_raw.w / ROI_raw.h) > 1.7777777) + { + ROI.x = ROI_raw.x; + ROI.w = ROI_raw.w; + ROI.h = (int) (ROI_raw.w * 0.5625 + .5); + + if ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) < 0) + { + ROI.y = 0; + ROI.h -= (int) ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) + .5); + } + else if ((ROI_raw.y + ROI_raw.h + .5 * (ROI.h - ROI_raw.h)) >= frame.rows) + { + ROI.y = (int) (ROI_raw.y - (frame.rows - 1 - (ROI_raw.y + .5 * (ROI.h - ROI_raw.h))) - .5); + ROI.h = frame.rows - 1; + } + else + { + ROI.y = (int) (ROI_raw.y - .5 * (ROI.h - ROI_raw.h) + .5); + } + } + + std::cout << "AFTER ASPECT RATIO CORRECTION" << std::endl; + std::cout << ROI.x << " " << ROI.y << " " << ROI.w << " " << ROI.h << std::endl; + + bb_ring_memory[framesCounter % bb_history] = ROI; + } + + /* + struct bounding_box ROI_raw; + ROI_raw = expZoomSmoothing(bb_ring_memory, framesCounter, 0.3); + + if (ROI_raw.x - 50 >= 0) ROI_raw.x -= 50; + if (ROI_raw.y - 50 >= 0) ROI_raw.y -= 50; + if (ROI_raw.x + ROI_raw.w + 100 <= frame.cols) ROI_raw.w += 100; + if (ROI_raw.y + ROI_raw.h + 100 <= frame.rows) ROI_raw.h += 100; + + struct bounding_box ROI; + + // check for and extablish 16:9 aspect ratio + // TODO: This probably needs testing and more case coverage! + std::cout << "x: "<< ROI_raw.x << ", y: " << ROI_raw.y << std::endl; + std::cout << ROI_raw.w << " x " << ROI_raw.h << std::endl; + if ((ROI_raw.w / ROI_raw.h) < 1.7777777) + { + ROI.y = ROI_raw.y; + ROI.w = (int) (ROI_raw.h * 1.7777777 + .5); + ROI.h = ROI_raw.h; + + if ((ROI_raw.x - .5 * (ROI.w - ROI_raw.w)) < 0) { + ROI.x = 0; + ROI.w -= (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); + } + else if ((ROI_raw.x + ROI_raw.w + .5 * (ROI.w - ROI_raw.w)) >= frame.cols) + { + ROI.x = (int) (ROI_raw.x - (frame.cols - 1 - (ROI_raw.x + .5 * (ROI.w - ROI_raw.w))) + .5); + ROI.w = frame.cols - 1; + } + else + { + ROI.x = (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); + } + } + else + if ((ROI_raw.w / ROI_raw.h) > 1.7777777) + { + ROI.x = ROI_raw.x; + ROI.w = ROI_raw.w; + ROI.h = (int) (ROI_raw.w * 0.5625 + .5); + + if ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) < 0) + { + ROI.y = 0; + ROI.h -= (int) ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) + .5); + std::cout << ROI.h << std::endl; + } + else if ((ROI_raw.y + ROI_raw.h + .5 * (ROI.h - ROI_raw.h)) >= frame.rows) + { + ROI.y = (int) (ROI_raw.y - (frame.rows - 1 - (ROI_raw.y + .5 * (ROI.h - ROI_raw.h))) + .5); + ROI.h = frame.rows - 1; + } + else + { + ROI.y = (int) (ROI_raw.y - .5 * (ROI.h - ROI_raw.h) + .5); + } + }*/ + + struct bounding_box smoothedROI; + smoothedROI = expZoomSmoothing(bb_ring_memory, framesCounter, 0.2); + + cv::Rect crop_region; + crop_region.x = smoothedROI.x; + crop_region.y = smoothedROI.y; + crop_region.width = smoothedROI.w; + crop_region.height = smoothedROI.h; + + if(crop_region.x < 0) + { + crop_region.width -= crop_region.x; + crop_region.x = 0; + } + if(crop_region.y < 0) + { + crop_region.height -= crop_region.y; + crop_region.y = 0; + } + if(crop_region.x + crop_region.width > frame.cols - 1) + { + crop_region.x += frame.cols - 1 - (crop_region.x + crop_region.width); + } + if(crop_region.y + crop_region.height > frame.rows - 1) + { + crop_region.y += frame.rows - 1 - (crop_region.y + crop_region.rows); + } + + + std::cout << "SMOOTHING DONE" << std::endl; + std::cout << crop_region.x << " " << crop_region.y << " " << crop_region.width << " " << crop_region.height << std::endl; + std::cout << frame.cols << " x " << frame.rows << std::endl; + + + cv::Mat croppedImage = frame(crop_region); + // Scale ROI to frame size + cv::resize(croppedImage, croppedImage, cv::Size(frame.cols, frame.rows), 0, 0, cv::INTER_LINEAR); + + return croppedImage; + + } + + //--------------------------------------------------------// + //------------------- Smoothing --------------------------// + //--------------------------------------------------------// + + // Source: https://de.wikibooks.org/wiki/Statistik:_Gl%C3%A4ttungsverfahren + + /** + * Exponential Smoothing of Time Series Data, in this case an n-slot ring memory. + * Return value should be kept and stored for the next call, + * where it should be used as the 'lastSmoothedValue' Parameter. + * + * @param alpha float Smoothing-Faktor (0,..,1) + * @param lastSmoothedValue float Smoothing result for previous entry of time series + * @param value float Current value in time series to be smoothed + */ + float exp_smoothing (float alpha, float lastSmoothedValue, float value) + { + return alpha * value + (1-alpha) * lastSmoothedValue; + } + + /** + * Double the Exponential Smoothing + * + * @param alpha float Smoothing-Faktor (0,..,1) + * @param lastDoublySmoothedValue float Doubly smoothed result for previous entry of time series + * @param smoothedValue float Smoothed value of current entry in time series to be doubly smoothed + */ + float doubled_exp_smoothing (float alpha, float lastDoublySmoothedValue, float smoothedValue) + { + return alpha * smoothedValue + (1-alpha) * lastDoublySmoothedValue; + } + + /** + * Finish up double smoothing to find a useful prognosis value. + * + * @param smoothedValue float Smoothing result for current value in TS + * @param doublySmoothedValue float DoubleSmoothing result for curr value in TS + */ + float finish_double_smoothing (float smoothedValue, float doublySmoothedValue) + { + return std::max(0.0f, 2 * smoothedValue - doublySmoothedValue); + } + + /** + * Zoom Smoothing using Exponential Smoothing (double smoothing is used). + * The purpose of this method is to find reasonable window to crop out of a frame around a ROI; + * the crop region should be chosen to simulate a camera zoom, while avoiding to be shaky + * due to inaccurate or moving ROI measurements. + * + * @param array_of_BBs struct[] Ring Memory containing a struct (four-touple) holding BB coordinates (ROIs) + * @param frame_count uint Index of current frame's ROI in ring memory + * @param smoothing_fac float Smoothing factor alpha, value to be chosen from interval (0,..,1) + */ + struct bounding_box expZoomSmoothing (struct bounding_box array_of_BBs[], int frame_count, float smoothing_fav) + { + // x and y coordinates of a bounding boxes center + // w and h are width and height of the BB + float x = 0.0, y = 0.0, w = 0.0, h = 0.0; + + // initialize some values (timeseries needs to start with values furthest backwards) + struct bounding_box last_smoothed = array_of_BBs[(frame_count - (bb_history - 1)) % bb_history]; + struct bounding_box last_double_smoothed = array_of_BBs[(frame_count - (bb_history - 1)) % bb_history]; + + for (int i = bb_history-2; i > 0; --i) + { + struct bounding_box curr = array_of_BBs[(frame_count - i) % bb_history]; + + last_smoothed.x = exp_smoothing(smoothing_fav, last_smoothed.x, curr.x); + last_double_smoothed.x = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.x, last_smoothed.x); + + last_smoothed.y = exp_smoothing(smoothing_fav, last_smoothed.y, curr.y); + last_double_smoothed.y = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.y, last_smoothed.y); + + last_smoothed.w = exp_smoothing(smoothing_fav, last_smoothed.w, curr.w); + last_double_smoothed.w = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.w, last_smoothed.w); + + last_smoothed.h = exp_smoothing(smoothing_fav, last_smoothed.h, curr.h); + last_double_smoothed.h = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.h, last_smoothed.h); + } + + x = finish_double_smoothing(last_smoothed.x, last_double_smoothed.x); + y = finish_double_smoothing(last_smoothed.y, last_double_smoothed.y); + w = finish_double_smoothing(last_smoothed.w, last_double_smoothed.w); + h = finish_double_smoothing(last_smoothed.h, last_double_smoothed.h); + + // eventuell kein finish und stattdessen lastSmoothedValues verwenden (kein doublen) + // je nach performance kann ein Teil zuvor als Mean berechnet werden und als "kopfelement" + // in der exponentiellen Glättung verwendet werden. Aber das müssen wir testen. + + return bounding_box {int(x), int(y), int(w), int(h)}; + } }; #endif From 29ec1c15344a416d98cdf4198f25cbcac0396cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Wed, 19 Feb 2020 08:37:08 +0100 Subject: [PATCH 6/8] Minor Bugfixes, more Stability, Update .gitignore --- .gitignore | 4 +++ example/examples.h | 74 ++++++---------------------------------------- 2 files changed, 13 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index cee3bf18c..5378384aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ build CMakeLists.txt.user* +cmake-build-debug + +# IDE +.idea # SmartCameraOperator for Horses and Riders # Data, Weights, other stuff diff --git a/example/examples.h b/example/examples.h index 441a9833c..80dfa35e0 100644 --- a/example/examples.h +++ b/example/examples.h @@ -847,7 +847,7 @@ class YoloDarknetExample : public VideoExample // ---------------------------------------------------------------------- // // ---------------------------------------------------------------------- // -#define bb_history 100 +#define bb_history 50 #ifdef BUILD_YOLO_LIB // ---------------------------------------------------------------------- @@ -1037,7 +1037,7 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample else { frames_without_detected_objects = 0; - struct bounding_box joint_ROI {frame.cols, frame.rows, 0, 0}; + struct bounding_box joint_ROI {frame.cols, frame.rows, 1, 1}; for (const auto& track : tracks) { cv::Rect brect = track.m_rrect.boundingRect(); @@ -1127,73 +1127,16 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample } } - std::cout << "AFTER ASPECT RATIO CORRECTION" << std::endl; - std::cout << ROI.x << " " << ROI.y << " " << ROI.w << " " << ROI.h << std::endl; - bb_ring_memory[framesCounter % bb_history] = ROI; } - /* - struct bounding_box ROI_raw; - ROI_raw = expZoomSmoothing(bb_ring_memory, framesCounter, 0.3); - - if (ROI_raw.x - 50 >= 0) ROI_raw.x -= 50; - if (ROI_raw.y - 50 >= 0) ROI_raw.y -= 50; - if (ROI_raw.x + ROI_raw.w + 100 <= frame.cols) ROI_raw.w += 100; - if (ROI_raw.y + ROI_raw.h + 100 <= frame.rows) ROI_raw.h += 100; - - struct bounding_box ROI; + struct bounding_box smoothedROI; + smoothedROI = expZoomSmoothing(bb_ring_memory, framesCounter, 0.2); - // check for and extablish 16:9 aspect ratio - // TODO: This probably needs testing and more case coverage! - std::cout << "x: "<< ROI_raw.x << ", y: " << ROI_raw.y << std::endl; - std::cout << ROI_raw.w << " x " << ROI_raw.h << std::endl; - if ((ROI_raw.w / ROI_raw.h) < 1.7777777) + if (smoothedROI.h == 0 || smoothedROI.w == 0) { - ROI.y = ROI_raw.y; - ROI.w = (int) (ROI_raw.h * 1.7777777 + .5); - ROI.h = ROI_raw.h; - - if ((ROI_raw.x - .5 * (ROI.w - ROI_raw.w)) < 0) { - ROI.x = 0; - ROI.w -= (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); - } - else if ((ROI_raw.x + ROI_raw.w + .5 * (ROI.w - ROI_raw.w)) >= frame.cols) - { - ROI.x = (int) (ROI_raw.x - (frame.cols - 1 - (ROI_raw.x + .5 * (ROI.w - ROI_raw.w))) + .5); - ROI.w = frame.cols - 1; - } - else - { - ROI.x = (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); - } + smoothedROI = bb_ring_memory[framesCounter % bb_history]; } - else - if ((ROI_raw.w / ROI_raw.h) > 1.7777777) - { - ROI.x = ROI_raw.x; - ROI.w = ROI_raw.w; - ROI.h = (int) (ROI_raw.w * 0.5625 + .5); - - if ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) < 0) - { - ROI.y = 0; - ROI.h -= (int) ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) + .5); - std::cout << ROI.h << std::endl; - } - else if ((ROI_raw.y + ROI_raw.h + .5 * (ROI.h - ROI_raw.h)) >= frame.rows) - { - ROI.y = (int) (ROI_raw.y - (frame.rows - 1 - (ROI_raw.y + .5 * (ROI.h - ROI_raw.h))) + .5); - ROI.h = frame.rows - 1; - } - else - { - ROI.y = (int) (ROI_raw.y - .5 * (ROI.h - ROI_raw.h) + .5); - } - }*/ - - struct bounding_box smoothedROI; - smoothedROI = expZoomSmoothing(bb_ring_memory, framesCounter, 0.2); cv::Rect crop_region; crop_region.x = smoothedROI.x; @@ -1201,6 +1144,7 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample crop_region.width = smoothedROI.w; crop_region.height = smoothedROI.h; + if(crop_region.x < 0) { crop_region.width -= crop_region.x; @@ -1217,15 +1161,15 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample } if(crop_region.y + crop_region.height > frame.rows - 1) { - crop_region.y += frame.rows - 1 - (crop_region.y + crop_region.rows); + crop_region.y += frame.rows - 1 - (crop_region.y + crop_region.height); } - std::cout << "SMOOTHING DONE" << std::endl; std::cout << crop_region.x << " " << crop_region.y << " " << crop_region.width << " " << crop_region.height << std::endl; std::cout << frame.cols << " x " << frame.rows << std::endl; + cv::Mat croppedImage = frame(crop_region); // Scale ROI to frame size cv::resize(croppedImage, croppedImage, cv::Size(frame.cols, frame.rows), 0, 0, cv::INTER_LINEAR); From 93152f3ba2ad3259dc570c186f2cf8eedea748c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Wed, 11 Mar 2020 13:33:05 +0100 Subject: [PATCH 7/8] Add Methods for SmartCamOperator Speed-Up under assumption of solved I/O bottleneck --- example/VideoExample.cpp | 223 +++++++++++++++++++++++++++++++++++++++ example/VideoExample.h | 3 + example/examples.h | 7 +- example/main.cpp | 2 +- 4 files changed, 228 insertions(+), 7 deletions(-) diff --git a/example/VideoExample.cpp b/example/VideoExample.cpp index 2cace030e..fdf794d5a 100644 --- a/example/VideoExample.cpp +++ b/example/VideoExample.cpp @@ -331,6 +331,7 @@ void VideoExample::CaptureAndDetect(VideoExample* thisPtr, std::atomic& st } } + int64 t1 = cv::getTickCount(); thisPtr->Detection(frameInfo.m_frame, frameInfo.m_regions); int64 t2 = cv::getTickCount(); @@ -371,6 +372,228 @@ void VideoExample::Detection(cv::Mat frame, regions_t& regions) regions.assign(std::begin(regs), std::end(regs)); } + +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Speed-Up by skipping frame in Detection - Attempted Tradeoff between accuracy and efficiency! + * Altered Methods specifically for SmartCameraOperator_w_YOLODetector + */ + +/// +/// \brief VideoExample::AsyncProcess +/// +void VideoExample::AsyncProcess_SmartCamOp() +{ + std::atomic stopCapture(false); + + std::thread thCapDet(CaptureAndDetect_SmartCamOp, this, std::ref(stopCapture)); + + cv::VideoWriter writer; + cv::VideoWriter writerZoom; + +#ifndef SILENT_WORK + cv::namedWindow("Video", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO); + bool manualMode = false; +#endif + + double freq = cv::getTickFrequency(); + + int framesCounter = m_startFrame + 1; + + int64 allTime = 0; + int64 startLoopTime = cv::getTickCount(); + size_t processCounter = 0; + for (; !stopCapture.load(); ) + { + FrameInfo& frameInfo = m_frameInfo[processCounter % 2]; + { + std::unique_lock lock(frameInfo.m_mutex); + if (!frameInfo.m_cond.wait_for(lock, std::chrono::milliseconds(m_captureTimeOut), [&frameInfo]{ return frameInfo.m_captured; })) + { + std::cout << "Wait frame timeout!" << std::endl; + break; + } + } + + if (!m_isTrackerInitialized) + { + cv::UMat ufirst = frameInfo.m_frame.getUMat(cv::ACCESS_READ); + m_isTrackerInitialized = InitTracker(ufirst); + if (!m_isTrackerInitialized) + { + std::cerr << "CaptureAndDetect: Tracker initialize error!!!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + } + + int64 t1 = cv::getTickCount(); + + Tracking(frameInfo.m_frame, frameInfo.m_regions); + + int64 t2 = cv::getTickCount(); + + allTime += t2 - t1 + frameInfo.m_dt; + int currTime = cvRound(1000 * (t2 - t1 + frameInfo.m_dt) / freq); + + //std::cout << "Frame " << framesCounter << ": td = " << (1000 * frameInfo.m_dt / freq) << ", tt = " << (1000 * (t2 - t1) / freq) << std::endl; + + // Crop, Zoom, and Save to new Video file + cv::Mat zoomedFrame = ZoomInOnROI(frameInfo.m_frame, framesCounter, currTime); + WriteZoomedFrame(writerZoom, zoomedFrame); + + DrawData(frameInfo.m_frame, framesCounter, currTime); + + WriteFrame(writer, frameInfo.m_frame); + + int k = 0; + +#ifndef SILENT_WORK + cv::imshow("Video", frameInfo.m_frame); + + int waitTime = manualMode ? 0 : 1;// std::max(1, cvRound(1000 / m_fps - currTime)); + k = cv::waitKey(waitTime); + if (k == 'm' || k == 'M') + { + manualMode = !manualMode; + } +#else + std::this_thread::sleep_for(std::chrono::milliseconds(1)); +#endif + + { + std::unique_lock lock(frameInfo.m_mutex); + frameInfo.m_captured = false; + } + frameInfo.m_cond.notify_one(); + + if (k == 27) + { + break; + } + ++framesCounter; + if (m_endFrame && framesCounter > m_endFrame) + { + std::cout << "Process: riched last " << m_endFrame << " frame" << std::endl; + break; + } + ++processCounter; + } + stopCapture = true; + + if (thCapDet.joinable()) + { + thCapDet.join(); + } + + int64 stopLoopTime = cv::getTickCount(); + + std::cout << "algorithms time = " << (allTime / freq) << ", work time = " << ((stopLoopTime - startLoopTime) / freq) << std::endl; + +#ifndef SILENT_WORK + cv::waitKey(m_finishDelay); +#endif +} + +/// +/// \brief VideoExample::CaptureAndDetect +/// \param thisPtr +/// \param stopCapture +/// +void VideoExample::CaptureAndDetect_SmartCamOp(VideoExample* thisPtr, std::atomic& stopCapture) +{ + cv::VideoCapture capture; + if (!thisPtr->OpenCapture(capture)) + { + std::cerr << "Can't open " << thisPtr->m_inFile << std::endl; + stopCapture = true; + return; + } + + int trackingTimeOut = thisPtr->m_trackingTimeOut; + size_t processCounter = 0; + for (; !stopCapture.load();) + { + FrameInfo& frameInfo = thisPtr->m_frameInfo[processCounter % 2]; + + { + std::unique_lock lock(frameInfo.m_mutex); + if (!frameInfo.m_cond.wait_for(lock, std::chrono::milliseconds(trackingTimeOut), [&frameInfo]{ return !frameInfo.m_captured; })) + { + std::cout << "Wait tracking timeout!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + } + + capture >> frameInfo.m_frame; + if (frameInfo.m_frame.empty()) + { + std::cerr << "CaptureAndDetect: frame is empty!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + + if (!thisPtr->m_isDetectorInitialized) + { + cv::UMat ufirst = frameInfo.m_frame.getUMat(cv::ACCESS_READ); + thisPtr->m_isDetectorInitialized = thisPtr->InitDetector(ufirst); + if (!thisPtr->m_isDetectorInitialized) + { + std::cerr << "CaptureAndDetect: Detector initialize error!!!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + } + + int64 t1 = cv::getTickCount(); + thisPtr->Detection_SmartCamOp(frameInfo.m_frame, frameInfo.m_regions, processCounter); + int64 t2 = cv::getTickCount(); + frameInfo.m_dt = t2 - t1; + + { + std::unique_lock lock(frameInfo.m_mutex); + frameInfo.m_captured = true; + } + frameInfo.m_cond.notify_one(); + + ++processCounter; + } + stopCapture = true; +} + + +/// +/// \brief VideoExample::Detection +/// \param frame +/// \param regions +/// +void VideoExample::Detection_SmartCamOp(cv::Mat frame, regions_t& regions, int frameCounter) +{ + if(frameCounter % 1 == 0) { + cv::UMat uframe; + if (!m_detector->CanGrayProcessing()) { + uframe = frame.getUMat(cv::ACCESS_READ); + } else { + cv::cvtColor(frame, uframe, cv::COLOR_BGR2GRAY); + } + + m_detector->Detect(uframe); + + const regions_t ®s = m_detector->GetDetects(); + + regions.assign(std::begin(regs), std::end(regs)); + } +} + + +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// + /// /// \brief VideoExample::Tracking /// \param frame diff --git a/example/VideoExample.h b/example/VideoExample.h index 214028bdb..0cbbd833f 100644 --- a/example/VideoExample.h +++ b/example/VideoExample.h @@ -27,6 +27,7 @@ class VideoExample virtual ~VideoExample() = default; void AsyncProcess(); + void AsyncProcess_SmartCamOp(); void SyncProcess(); cv::UMat getFrame() @@ -45,11 +46,13 @@ class VideoExample int m_trackingTimeOut = 60000; static void CaptureAndDetect(VideoExample* thisPtr, std::atomic& stopCapture); + static void CaptureAndDetect_SmartCamOp(VideoExample* thisPtr, std::atomic& stopCapture); virtual bool InitDetector(cv::UMat frame) = 0; virtual bool InitTracker(cv::UMat frame) = 0; void Detection(cv::Mat frame, regions_t& regions); + void Detection_SmartCamOp(cv::Mat frame, regions_t& regions, int frameCounter); void Tracking(cv::Mat frame, const regions_t& regions); virtual void DrawData(cv::Mat frame, int framesCounter, int currTime) = 0; diff --git a/example/examples.h b/example/examples.h index 80dfa35e0..14f155787 100644 --- a/example/examples.h +++ b/example/examples.h @@ -850,7 +850,7 @@ class YoloDarknetExample : public VideoExample #define bb_history 50 #ifdef BUILD_YOLO_LIB -// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- // /// /// \brief The SmartCamOperator_w_YoloDarknet class @@ -1165,11 +1165,6 @@ class SmartCamOperator_w_YoloDarknet : public VideoExample } - std::cout << crop_region.x << " " << crop_region.y << " " << crop_region.width << " " << crop_region.height << std::endl; - std::cout << frame.cols << " x " << frame.rows << std::endl; - - - cv::Mat croppedImage = frame(crop_region); // Scale ROI to frame size cv::resize(croppedImage, croppedImage, cv::Size(frame.cols, frame.rows), 0, 0, cv::INTER_LINEAR); diff --git a/example/main.cpp b/example/main.cpp index 9a04a4ce8..157f7935d 100644 --- a/example/main.cpp +++ b/example/main.cpp @@ -97,7 +97,7 @@ int main(int argc, char** argv) case 7: { SmartCamOperator_w_YoloDarknet smart_cam_operator(parser); - asyncPipeline ? smart_cam_operator.AsyncProcess() : smart_cam_operator.SyncProcess(); + asyncPipeline ? smart_cam_operator.AsyncProcess_SmartCamOp() : smart_cam_operator.SyncProcess(); break; } #endif From 96d645e82dd7c31032b851a172fe1e148c75eb7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20K=C3=BChnel?= Date: Wed, 8 Sep 2021 13:02:38 +0200 Subject: [PATCH 8/8] Minor Edit --- example/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/main.cpp b/example/main.cpp index 157f7935d..eea07691f 100644 --- a/example/main.cpp +++ b/example/main.cpp @@ -10,7 +10,7 @@ static void Help() { printf("\nExamples of the Multitarget tracking algorithm\n" "Usage: \n" - " ./MultitargetTracker [--example]= [--start_frame]= [--end_frame]= [--end_delay]= [--out]= [--show_logs]= [--async]= \n\n" + " ./MultitargetTracker [--example]= [--start_frame]= [--end_frame]= [--end_delay]= [--out]= [--show_logs]= [--async]= \n\n" "Press:\n" "\'m\' key for change mode: play|pause. When video is paused you can press any key for get next frame. \n\n" "Press Esc to exit from video \n\n" @@ -20,7 +20,7 @@ static void Help() const char* keys = { "{ @1 |../data/atrium.avi | movie file | }" - "{ e example |1 | number of example 0 - MouseTracking, 1 - MotionDetector, 2 - FaceDetector, 3 - PedestrianDetector, 4 - MobileNet SSD detector, 5 - Yolo OpenCV detector, 6 - Yolo Darknet detector | }" + "{ e example |1 | number of example 0 - MouseTracking, 1 - MotionDetector, 2 - FaceDetector, 3 - PedestrianDetector, 4 - MobileNet SSD detector, 5 - Yolo OpenCV detector, 6 - Yolo Darknet detector | 7 - SmartCamOperator w/ YoloDarknet | }" "{ sf start_frame |0 | Start a video from this position | }" "{ ef end_frame |0 | Play a video to this position (if 0 then played to the end of file) | }" "{ ed end_delay |0 | Delay in milliseconds after video ending | }"