diff --git a/.gitignore b/.gitignore index e3814ab6b..5378384aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,14 @@ build CMakeLists.txt.user* +cmake-build-debug +# IDE +.idea + +# SmartCameraOperator for Horses and Riders +# Data, Weights, other stuff + +data/Kirchhellen*.avi +data/*.names +data/yolov3.cfg +data/yolov3.weights diff --git a/CMakeLists.txt b/CMakeLists.txt index f30454e8d..c1849f1b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,10 +12,10 @@ if (OPENMP_FOUND) list(APPEND CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) if (CMAKE_COMPILER_IS_GNUCXX) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic-errors -std=c++14" CACHE STRING COMPILE_FLAGS FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic-errors -std=c++17 -lstdc++fs" CACHE STRING COMPILE_FLAGS FORCE) set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -march=native -mtune=native -funroll-loops -Wall -DNDEBUG -DBOOST_DISABLE_ASSERTS" CACHE STRING COMPILE_FLAGS FORCE) set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -march=native -mtune=native -Wall -DDEBUG" CACHE STRING COMPILE_FLAGS FORCE) elseif (MSVC) @@ -45,12 +45,12 @@ if (BUILD_CARS_COUNTING) add_subdirectory(cars_counting) endif(BUILD_CARS_COUNTING) -option(BUILD_ASYNC_DETECTOR "Should compiled async example with low fps Detector?" OFF) +option(BUILD_ASYNC_DETECTOR "Should compiled async example with low fps Detector?" ON) if (BUILD_ASYNC_DETECTOR) add_subdirectory(async_detector) endif(BUILD_ASYNC_DETECTOR) -option(BUILD_YOLO_LIB "Should compiled standalone yolo_lib with original darknet?" OFF) +option(BUILD_YOLO_LIB "Should compiled standalone yolo_lib with original darknet?" ON) if (BUILD_YOLO_LIB) add_subdirectory(src/Detector/darknet) add_definitions(-DBUILD_YOLO_LIB) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b57f99e0a..c44b94d52 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -47,3 +47,4 @@ endif(BUILD_YOLO_LIB) ADD_EXECUTABLE(${PROJECT_NAME} ${SOURCES} ${HEADERS}) TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${LIBS}) +target_link_libraries(${PROJECT_NAME} /usr/lib/gcc/x86_64-linux-gnu/7/libstdc++fs.a) diff --git a/example/VideoExample.cpp b/example/VideoExample.cpp index de8dc32a1..fdf794d5a 100644 --- a/example/VideoExample.cpp +++ b/example/VideoExample.cpp @@ -2,6 +2,10 @@ #include #include +#include +#include +namespace fs = std::experimental::filesystem; + #include "VideoExample.h" /// @@ -23,6 +27,16 @@ VideoExample::VideoExample(const cv::CommandLineParser& parser) { m_inFile = parser.get(0); m_outFile = parser.get("out"); + + fs::path path(parser.get("out")); + auto filename = path.stem(); + filename += "_zoomed"; + auto ext = path.extension(); + filename += ext.string(); + auto parent = path.parent_path(); + path = parent / filename; + m_outFileZoomed = path.string(); + m_showLogs = parser.get("show_logs") != 0; m_startFrame = parser.get("start_frame"); m_endFrame = parser.get("end_frame"); @@ -45,6 +59,7 @@ VideoExample::VideoExample(const cv::CommandLineParser& parser) void VideoExample::SyncProcess() { cv::VideoWriter writer; + cv::VideoWriter writerZoom; #ifndef SILENT_WORK cv::namedWindow("Video", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO); @@ -109,6 +124,10 @@ void VideoExample::SyncProcess() allTime += t2 - t1; int currTime = cvRound(1000 * (t2 - t1) / freq); + // Crop, Zoom, and Save to new Video file + cv::Mat zoomedFrame = ZoomInOnROI(frame, framesCounter, currTime); + WriteZoomedFrame(writerZoom, zoomedFrame); + DrawData(frame, framesCounter, currTime); #ifndef SILENT_WORK @@ -156,6 +175,7 @@ void VideoExample::AsyncProcess() std::thread thCapDet(CaptureAndDetect, this, std::ref(stopCapture)); cv::VideoWriter writer; + cv::VideoWriter writerZoom; #ifndef SILENT_WORK cv::namedWindow("Video", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO); @@ -204,6 +224,10 @@ void VideoExample::AsyncProcess() //std::cout << "Frame " << framesCounter << ": td = " << (1000 * frameInfo.m_dt / freq) << ", tt = " << (1000 * (t2 - t1) / freq) << std::endl; + // Crop, Zoom, and Save to new Video file + cv::Mat zoomedFrame = ZoomInOnROI(frameInfo.m_frame, framesCounter, currTime); + WriteZoomedFrame(writerZoom, zoomedFrame); + DrawData(frameInfo.m_frame, framesCounter, currTime); WriteFrame(writer, frameInfo.m_frame); @@ -307,6 +331,7 @@ void VideoExample::CaptureAndDetect(VideoExample* thisPtr, std::atomic& st } } + int64 t1 = cv::getTickCount(); thisPtr->Detection(frameInfo.m_frame, frameInfo.m_regions); int64 t2 = cv::getTickCount(); @@ -347,6 +372,228 @@ void VideoExample::Detection(cv::Mat frame, regions_t& regions) regions.assign(std::begin(regs), std::end(regs)); } + +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Speed-Up by skipping frame in Detection - Attempted Tradeoff between accuracy and efficiency! + * Altered Methods specifically for SmartCameraOperator_w_YOLODetector + */ + +/// +/// \brief VideoExample::AsyncProcess +/// +void VideoExample::AsyncProcess_SmartCamOp() +{ + std::atomic stopCapture(false); + + std::thread thCapDet(CaptureAndDetect_SmartCamOp, this, std::ref(stopCapture)); + + cv::VideoWriter writer; + cv::VideoWriter writerZoom; + +#ifndef SILENT_WORK + cv::namedWindow("Video", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO); + bool manualMode = false; +#endif + + double freq = cv::getTickFrequency(); + + int framesCounter = m_startFrame + 1; + + int64 allTime = 0; + int64 startLoopTime = cv::getTickCount(); + size_t processCounter = 0; + for (; !stopCapture.load(); ) + { + FrameInfo& frameInfo = m_frameInfo[processCounter % 2]; + { + std::unique_lock lock(frameInfo.m_mutex); + if (!frameInfo.m_cond.wait_for(lock, std::chrono::milliseconds(m_captureTimeOut), [&frameInfo]{ return frameInfo.m_captured; })) + { + std::cout << "Wait frame timeout!" << std::endl; + break; + } + } + + if (!m_isTrackerInitialized) + { + cv::UMat ufirst = frameInfo.m_frame.getUMat(cv::ACCESS_READ); + m_isTrackerInitialized = InitTracker(ufirst); + if (!m_isTrackerInitialized) + { + std::cerr << "CaptureAndDetect: Tracker initialize error!!!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + } + + int64 t1 = cv::getTickCount(); + + Tracking(frameInfo.m_frame, frameInfo.m_regions); + + int64 t2 = cv::getTickCount(); + + allTime += t2 - t1 + frameInfo.m_dt; + int currTime = cvRound(1000 * (t2 - t1 + frameInfo.m_dt) / freq); + + //std::cout << "Frame " << framesCounter << ": td = " << (1000 * frameInfo.m_dt / freq) << ", tt = " << (1000 * (t2 - t1) / freq) << std::endl; + + // Crop, Zoom, and Save to new Video file + cv::Mat zoomedFrame = ZoomInOnROI(frameInfo.m_frame, framesCounter, currTime); + WriteZoomedFrame(writerZoom, zoomedFrame); + + DrawData(frameInfo.m_frame, framesCounter, currTime); + + WriteFrame(writer, frameInfo.m_frame); + + int k = 0; + +#ifndef SILENT_WORK + cv::imshow("Video", frameInfo.m_frame); + + int waitTime = manualMode ? 0 : 1;// std::max(1, cvRound(1000 / m_fps - currTime)); + k = cv::waitKey(waitTime); + if (k == 'm' || k == 'M') + { + manualMode = !manualMode; + } +#else + std::this_thread::sleep_for(std::chrono::milliseconds(1)); +#endif + + { + std::unique_lock lock(frameInfo.m_mutex); + frameInfo.m_captured = false; + } + frameInfo.m_cond.notify_one(); + + if (k == 27) + { + break; + } + ++framesCounter; + if (m_endFrame && framesCounter > m_endFrame) + { + std::cout << "Process: riched last " << m_endFrame << " frame" << std::endl; + break; + } + ++processCounter; + } + stopCapture = true; + + if (thCapDet.joinable()) + { + thCapDet.join(); + } + + int64 stopLoopTime = cv::getTickCount(); + + std::cout << "algorithms time = " << (allTime / freq) << ", work time = " << ((stopLoopTime - startLoopTime) / freq) << std::endl; + +#ifndef SILENT_WORK + cv::waitKey(m_finishDelay); +#endif +} + +/// +/// \brief VideoExample::CaptureAndDetect +/// \param thisPtr +/// \param stopCapture +/// +void VideoExample::CaptureAndDetect_SmartCamOp(VideoExample* thisPtr, std::atomic& stopCapture) +{ + cv::VideoCapture capture; + if (!thisPtr->OpenCapture(capture)) + { + std::cerr << "Can't open " << thisPtr->m_inFile << std::endl; + stopCapture = true; + return; + } + + int trackingTimeOut = thisPtr->m_trackingTimeOut; + size_t processCounter = 0; + for (; !stopCapture.load();) + { + FrameInfo& frameInfo = thisPtr->m_frameInfo[processCounter % 2]; + + { + std::unique_lock lock(frameInfo.m_mutex); + if (!frameInfo.m_cond.wait_for(lock, std::chrono::milliseconds(trackingTimeOut), [&frameInfo]{ return !frameInfo.m_captured; })) + { + std::cout << "Wait tracking timeout!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + } + + capture >> frameInfo.m_frame; + if (frameInfo.m_frame.empty()) + { + std::cerr << "CaptureAndDetect: frame is empty!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + + if (!thisPtr->m_isDetectorInitialized) + { + cv::UMat ufirst = frameInfo.m_frame.getUMat(cv::ACCESS_READ); + thisPtr->m_isDetectorInitialized = thisPtr->InitDetector(ufirst); + if (!thisPtr->m_isDetectorInitialized) + { + std::cerr << "CaptureAndDetect: Detector initialize error!!!" << std::endl; + frameInfo.m_cond.notify_one(); + break; + } + } + + int64 t1 = cv::getTickCount(); + thisPtr->Detection_SmartCamOp(frameInfo.m_frame, frameInfo.m_regions, processCounter); + int64 t2 = cv::getTickCount(); + frameInfo.m_dt = t2 - t1; + + { + std::unique_lock lock(frameInfo.m_mutex); + frameInfo.m_captured = true; + } + frameInfo.m_cond.notify_one(); + + ++processCounter; + } + stopCapture = true; +} + + +/// +/// \brief VideoExample::Detection +/// \param frame +/// \param regions +/// +void VideoExample::Detection_SmartCamOp(cv::Mat frame, regions_t& regions, int frameCounter) +{ + if(frameCounter % 1 == 0) { + cv::UMat uframe; + if (!m_detector->CanGrayProcessing()) { + uframe = frame.getUMat(cv::ACCESS_READ); + } else { + cv::cvtColor(frame, uframe, cv::COLOR_BGR2GRAY); + } + + m_detector->Detect(uframe); + + const regions_t ®s = m_detector->GetDetects(); + + regions.assign(std::begin(regs), std::end(regs)); + } +} + + +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////////////////////// + /// /// \brief VideoExample::Tracking /// \param frame @@ -472,3 +719,27 @@ bool VideoExample::WriteFrame(cv::VideoWriter& writer, const cv::Mat& frame) } return false; } + +/// +/// \brief VideoExample::WriteFrame +/// \param writer +/// \param frame +/// \return +/// +bool VideoExample::WriteZoomedFrame(cv::VideoWriter& writer, const cv::Mat& frame) +{ + if (!m_outFile.empty()) + { + if (!writer.isOpened()) + { + + writer.open(m_outFileZoomed, m_fourcc, m_fps, frame.size(), true); + } + if (writer.isOpened()) + { + writer << frame; + return true; + } + } + return false; +} diff --git a/example/VideoExample.h b/example/VideoExample.h index 24ed7aaf1..0cbbd833f 100644 --- a/example/VideoExample.h +++ b/example/VideoExample.h @@ -27,8 +27,14 @@ class VideoExample virtual ~VideoExample() = default; void AsyncProcess(); + void AsyncProcess_SmartCamOp(); void SyncProcess(); + cv::UMat getFrame() + { + return this->m_frameInfo[0].m_frame.getUMat(cv::ACCESS_READ); + } + protected: std::unique_ptr m_detector; std::unique_ptr m_tracker; @@ -40,15 +46,19 @@ class VideoExample int m_trackingTimeOut = 60000; static void CaptureAndDetect(VideoExample* thisPtr, std::atomic& stopCapture); + static void CaptureAndDetect_SmartCamOp(VideoExample* thisPtr, std::atomic& stopCapture); virtual bool InitDetector(cv::UMat frame) = 0; virtual bool InitTracker(cv::UMat frame) = 0; void Detection(cv::Mat frame, regions_t& regions); + void Detection_SmartCamOp(cv::Mat frame, regions_t& regions, int frameCounter); void Tracking(cv::Mat frame, const regions_t& regions); virtual void DrawData(cv::Mat frame, int framesCounter, int currTime) = 0; + virtual cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) = 0; + void DrawTrack(cv::Mat frame, int resizeCoeff, const TrackingObject& track, bool drawTrajectory = true); private: @@ -56,6 +66,7 @@ class VideoExample bool m_isDetectorInitialized = false; std::string m_inFile; std::string m_outFile; + std::string m_outFileZoomed; int m_fourcc = cv::VideoWriter::fourcc('M', 'J', 'P', 'G'); int m_startFrame = 0; int m_endFrame = 0; @@ -76,4 +87,5 @@ class VideoExample bool OpenCapture(cv::VideoCapture& capture); bool WriteFrame(cv::VideoWriter& writer, const cv::Mat& frame); + bool WriteZoomedFrame(cv::VideoWriter& writer, const cv::Mat& frame); }; diff --git a/example/examples.h b/example/examples.h index ad881dd09..14f155787 100644 --- a/example/examples.h +++ b/example/examples.h @@ -1,11 +1,22 @@ #pragma once #include +#include #include #include +#include + #include "VideoExample.h" +struct bounding_box +{ + int x; + int y; + int w; + int h; +}; + /// /// \brief DrawFilledRect /// @@ -154,6 +165,22 @@ class MotionDetectorExample : public VideoExample m_detector->CalcMotionMap(frame); } + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } private: int m_minObjWidth = 8; int m_minStaticTime = 5; @@ -250,6 +277,23 @@ class FaceDetectorExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; // ---------------------------------------------------------------------- @@ -347,6 +391,23 @@ class PedestrianDetectorExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; // ---------------------------------------------------------------------- @@ -453,6 +514,23 @@ class SSDMobileNetExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; // ---------------------------------------------------------------------- @@ -574,6 +652,23 @@ class YoloExample : public VideoExample m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } }; #ifdef BUILD_YOLO_LIB @@ -724,6 +819,451 @@ class YoloDarknetExample : public VideoExample //m_detector->CalcMotionMap(frame); } + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + if (framesCounter + currTime == 0){ + return frame; + } + else + { + return frame; + } + } +}; + +#endif + + +// ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // +// -------------- Smart Camera Operator Example Settings ---------------- // +// ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // + +#define bb_history 50 + +#ifdef BUILD_YOLO_LIB +// ---------------------------------------------------------------------- // + +/// +/// \brief The SmartCamOperator_w_YoloDarknet class +/// +class SmartCamOperator_w_YoloDarknet : public VideoExample +{ +public: + SmartCamOperator_w_YoloDarknet(const cv::CommandLineParser& parser) + : + VideoExample(parser) + { + + } + +protected: + + /// + /// \brief Ring Memory + /// + struct bounding_box bb_ring_memory[bb_history]; + + int frames_since_init = 0; + int frames_without_detected_objects = 0; + + /// + /// \brief InitDetector + /// \param frame + /// \return + /// + bool InitDetector(cv::UMat frame) + { + config_t config; + +#ifdef _WIN32 + std::string pathToModel = "../../data/"; +#else + std::string pathToModel = "../data/"; +#endif +#if 0 + config.emplace("modelConfiguration", pathToModel + "yolov3-tiny.cfg"); + config.emplace("modelBinary", pathToModel + "yolov3-tiny.weights"); + config.emplace("confidenceThreshold", "0.5"); +#else + config.emplace("modelConfiguration", pathToModel + "yolov3.cfg"); + config.emplace("modelBinary", pathToModel + "yolov3.weights"); + config.emplace("confidenceThreshold", "0.6"); +#endif + config.emplace("classNames", pathToModel + "horse_and_rider.names"); + config.emplace("maxCropRatio", "-1"); + + config.emplace("white_list", "Horse&Rider"); + + m_detector = std::unique_ptr(CreateDetector(tracking::Detectors::Yolo_Darknet, config, frame)); + if (m_detector.get()) + { + m_detector->SetMinObjectSize(cv::Size(frame.cols / 40, frame.rows / 40)); + return true; + } + return false; + } + + /// + /// \brief InitTracker + /// \param frame + /// \return + /// + bool InitTracker(cv::UMat frame) + { + TrackerSettings settings; + settings.SetDistance(tracking::DistCenters); + settings.m_kalmanType = tracking::KalmanLinear; + settings.m_filterGoal = tracking::FilterCenter; + settings.m_lostTrackType = tracking::TrackKCF; // Use visual objects tracker for collisions resolving + settings.m_matchType = tracking::MatchHungrian; + settings.m_dt = 0.3f; // Delta time for Kalman filter + settings.m_accelNoiseMag = 0.2f; // Accel noise magnitude for Kalman filter + settings.m_distThres = 0.8f; // Distance threshold between region and object on two frames + settings.m_minAreaRadius = frame.rows / 20.f; + settings.m_maximumAllowedSkippedFrames = cvRound(.8 * m_fps); // Maximum allowed skipped frames + settings.m_maxTraceLength = cvRound(2 * m_fps); // Maximum trace length + + /* + settings.AddNearTypes("car", "bus", false); + settings.AddNearTypes("car", "truck", false); + settings.AddNearTypes("person", "bicycle", true); + settings.AddNearTypes("person", "motorbike", true); + */ + + m_tracker = std::make_unique(settings); + + // Init RingMemory to show + for (int i = 0; i < bb_history; i++) + { + bb_ring_memory[i] = {0, 0, frame.cols-1, frame.rows-1}; + } + + return true; + } + + /// + /// \brief DrawData + /// \param frame + /// \param framesCounter + /// \param currTime + /// + void DrawData(cv::Mat frame, int framesCounter, int currTime) + { + auto tracks = m_tracker->GetTracks(); + + if (m_showLogs) + { + std::cout << "Frame " << framesCounter << ": tracks = " << tracks.size() << ", time = " << currTime << std::endl; + } + + for (const auto& track : tracks) + { + if (track.IsRobust(2, // Minimal trajectory size + 0.5f, // Minimal ratio raw_trajectory_points / trajectory_lenght + cv::Size2f(0.1f, 8.0f)) // Min and max ratio: width / height + ) + { + DrawTrack(frame, 1, track); + + std::stringstream label; + label << track.m_type << " " << std::setprecision(2) << track.m_velocity << ": " << track.m_confidence; + int baseLine = 0; + cv::Size labelSize = cv::getTextSize(label.str(), cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + cv::Rect brect = track.m_rrect.boundingRect(); + if (brect.x < 0) + { + brect.width = std::min(brect.width, frame.cols - 1); + brect.x = 0; + } + else if (brect.x + brect.width >= frame.cols) + { + brect.x = std::max(0, frame.cols - brect.width - 1); + brect.width = std::min(brect.width, frame.cols - 1); + } + if (brect.y - labelSize.height < 0) + { + brect.height = std::min(brect.height, frame.rows - 1); + brect.y = labelSize.height; + } + else if (brect.y + brect.height >= frame.rows) + { + brect.y = std::max(0, frame.rows - brect.height - 1); + brect.height = std::min(brect.height, frame.rows - 1); + } + DrawFilledRect(frame, cv::Rect(cv::Point(brect.x, brect.y - labelSize.height), cv::Size(labelSize.width, labelSize.height + baseLine)), cv::Scalar(200, 200, 200), 150); + cv::putText(frame, label.str(), brect.tl(), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); + } + } + + //m_detector->CalcMotionMap(frame); + } + + + /// + /// \brief ZoomInOnROI + /// \param frame + /// \param framesCounter + /// \param currTime + /// + cv::Mat ZoomInOnROI(cv::Mat frame, int framesCounter, int currTime) + { + auto tracks = m_tracker->GetTracks(); + + if (m_showLogs) + { + std::cout << "Frame " << framesCounter << ": tracks = " << tracks.size() << ", time = " << currTime << std::endl; + } + + if (tracks.size() == 0 && frames_without_detected_objects <= 100) + { + bb_ring_memory[framesCounter % bb_history] = bb_ring_memory[(framesCounter - 1) % bb_history]; + ++frames_without_detected_objects; + } + else if (tracks.size() == 0 && frames_without_detected_objects > 100) + { + struct bounding_box entire_frame {0, 0, frame.cols, frame.rows}; + bb_ring_memory[framesCounter % bb_history] = entire_frame; + } + else + { + frames_without_detected_objects = 0; + struct bounding_box joint_ROI {frame.cols, frame.rows, 1, 1}; + for (const auto& track : tracks) + { + cv::Rect brect = track.m_rrect.boundingRect(); + + if (brect.x < 0) + { + brect.width = std::min(brect.width, frame.cols - 1); + brect.x = 0; + } + else if ((brect.x + brect.width) >= frame.cols) + { + brect.x = std::max(0, frame.cols - brect.width - 1); + brect.width = std::min(brect.width, frame.cols - 1); + } + if (brect.y < 0) + { + brect.height = std::min(brect.height, frame.rows - 1); + brect.y = 0; + } + else if ((brect.y + brect.height) >= frame.rows) + { + brect.y = std::max(0, frame.rows - brect.height - 1); + brect.height = std::min(brect.height, frame.rows - 1); + } + joint_ROI.x = (brect.x < joint_ROI.x) ? brect.x : joint_ROI.x; + joint_ROI.y = (brect.y < joint_ROI.y) ? brect.y : joint_ROI.y; + joint_ROI.w = ((brect.x + brect.width) > joint_ROI.w) ? (brect.x + brect.width) : joint_ROI.w; + joint_ROI.h = ((brect.y + brect.height) > joint_ROI.h) ? (brect.y + brect.height) : joint_ROI.h; + + } + joint_ROI.w = joint_ROI.w - joint_ROI.x; + joint_ROI.h = joint_ROI.h - joint_ROI.y; + + struct bounding_box ROI_raw = joint_ROI; + + + if (ROI_raw.x - 50 >= 0) ROI_raw.x -= 50; + if (ROI_raw.y - 50 >= 0) ROI_raw.y -= 50; + if (ROI_raw.x + ROI_raw.w + 100 <= frame.cols) ROI_raw.w += 100; + if (ROI_raw.y + ROI_raw.h + 100 <= frame.rows) ROI_raw.h += 100; + + + struct bounding_box ROI; + + // check for and extablish 16:9 aspect ratio + // TODO: This probably needs testing and more case coverage! + if ((ROI_raw.w / ROI_raw.h) < 1.7777777) + { + ROI.y = ROI_raw.y; + ROI.w = (int) (ROI_raw.h * 1.7777777 + .5); + ROI.h = ROI_raw.h; + + if ((ROI_raw.x - .5 * (ROI.w - ROI_raw.w)) < 0) { + ROI.x = 0; + ROI.w -= (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); + } + else if ((ROI_raw.x + ROI_raw.w + .5 * (ROI.w - ROI_raw.w)) >= frame.cols) + { + ROI.x = (int) (ROI_raw.x - (frame.cols - 1 - (ROI_raw.x + .5 * (ROI.w - ROI_raw.w))) - .5); + ROI.w = frame.cols - 1; + } + else + { + ROI.x = (int) (ROI_raw.x - .5 * (ROI.w - ROI_raw.w) + .5); + } + } + else + if ((ROI_raw.w / ROI_raw.h) > 1.7777777) + { + ROI.x = ROI_raw.x; + ROI.w = ROI_raw.w; + ROI.h = (int) (ROI_raw.w * 0.5625 + .5); + + if ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) < 0) + { + ROI.y = 0; + ROI.h -= (int) ((ROI_raw.y - .5 * (ROI.h - ROI_raw.h)) + .5); + } + else if ((ROI_raw.y + ROI_raw.h + .5 * (ROI.h - ROI_raw.h)) >= frame.rows) + { + ROI.y = (int) (ROI_raw.y - (frame.rows - 1 - (ROI_raw.y + .5 * (ROI.h - ROI_raw.h))) - .5); + ROI.h = frame.rows - 1; + } + else + { + ROI.y = (int) (ROI_raw.y - .5 * (ROI.h - ROI_raw.h) + .5); + } + } + + bb_ring_memory[framesCounter % bb_history] = ROI; + } + + struct bounding_box smoothedROI; + smoothedROI = expZoomSmoothing(bb_ring_memory, framesCounter, 0.2); + + if (smoothedROI.h == 0 || smoothedROI.w == 0) + { + smoothedROI = bb_ring_memory[framesCounter % bb_history]; + } + + cv::Rect crop_region; + crop_region.x = smoothedROI.x; + crop_region.y = smoothedROI.y; + crop_region.width = smoothedROI.w; + crop_region.height = smoothedROI.h; + + + if(crop_region.x < 0) + { + crop_region.width -= crop_region.x; + crop_region.x = 0; + } + if(crop_region.y < 0) + { + crop_region.height -= crop_region.y; + crop_region.y = 0; + } + if(crop_region.x + crop_region.width > frame.cols - 1) + { + crop_region.x += frame.cols - 1 - (crop_region.x + crop_region.width); + } + if(crop_region.y + crop_region.height > frame.rows - 1) + { + crop_region.y += frame.rows - 1 - (crop_region.y + crop_region.height); + } + + + cv::Mat croppedImage = frame(crop_region); + // Scale ROI to frame size + cv::resize(croppedImage, croppedImage, cv::Size(frame.cols, frame.rows), 0, 0, cv::INTER_LINEAR); + + return croppedImage; + + } + + //--------------------------------------------------------// + //------------------- Smoothing --------------------------// + //--------------------------------------------------------// + + // Source: https://de.wikibooks.org/wiki/Statistik:_Gl%C3%A4ttungsverfahren + + /** + * Exponential Smoothing of Time Series Data, in this case an n-slot ring memory. + * Return value should be kept and stored for the next call, + * where it should be used as the 'lastSmoothedValue' Parameter. + * + * @param alpha float Smoothing-Faktor (0,..,1) + * @param lastSmoothedValue float Smoothing result for previous entry of time series + * @param value float Current value in time series to be smoothed + */ + float exp_smoothing (float alpha, float lastSmoothedValue, float value) + { + return alpha * value + (1-alpha) * lastSmoothedValue; + } + + /** + * Double the Exponential Smoothing + * + * @param alpha float Smoothing-Faktor (0,..,1) + * @param lastDoublySmoothedValue float Doubly smoothed result for previous entry of time series + * @param smoothedValue float Smoothed value of current entry in time series to be doubly smoothed + */ + float doubled_exp_smoothing (float alpha, float lastDoublySmoothedValue, float smoothedValue) + { + return alpha * smoothedValue + (1-alpha) * lastDoublySmoothedValue; + } + + /** + * Finish up double smoothing to find a useful prognosis value. + * + * @param smoothedValue float Smoothing result for current value in TS + * @param doublySmoothedValue float DoubleSmoothing result for curr value in TS + */ + float finish_double_smoothing (float smoothedValue, float doublySmoothedValue) + { + return std::max(0.0f, 2 * smoothedValue - doublySmoothedValue); + } + + /** + * Zoom Smoothing using Exponential Smoothing (double smoothing is used). + * The purpose of this method is to find reasonable window to crop out of a frame around a ROI; + * the crop region should be chosen to simulate a camera zoom, while avoiding to be shaky + * due to inaccurate or moving ROI measurements. + * + * @param array_of_BBs struct[] Ring Memory containing a struct (four-touple) holding BB coordinates (ROIs) + * @param frame_count uint Index of current frame's ROI in ring memory + * @param smoothing_fac float Smoothing factor alpha, value to be chosen from interval (0,..,1) + */ + struct bounding_box expZoomSmoothing (struct bounding_box array_of_BBs[], int frame_count, float smoothing_fav) + { + // x and y coordinates of a bounding boxes center + // w and h are width and height of the BB + float x = 0.0, y = 0.0, w = 0.0, h = 0.0; + + // initialize some values (timeseries needs to start with values furthest backwards) + struct bounding_box last_smoothed = array_of_BBs[(frame_count - (bb_history - 1)) % bb_history]; + struct bounding_box last_double_smoothed = array_of_BBs[(frame_count - (bb_history - 1)) % bb_history]; + + for (int i = bb_history-2; i > 0; --i) + { + struct bounding_box curr = array_of_BBs[(frame_count - i) % bb_history]; + + last_smoothed.x = exp_smoothing(smoothing_fav, last_smoothed.x, curr.x); + last_double_smoothed.x = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.x, last_smoothed.x); + + last_smoothed.y = exp_smoothing(smoothing_fav, last_smoothed.y, curr.y); + last_double_smoothed.y = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.y, last_smoothed.y); + + last_smoothed.w = exp_smoothing(smoothing_fav, last_smoothed.w, curr.w); + last_double_smoothed.w = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.w, last_smoothed.w); + + last_smoothed.h = exp_smoothing(smoothing_fav, last_smoothed.h, curr.h); + last_double_smoothed.h = doubled_exp_smoothing(smoothing_fav, last_double_smoothed.h, last_smoothed.h); + } + + x = finish_double_smoothing(last_smoothed.x, last_double_smoothed.x); + y = finish_double_smoothing(last_smoothed.y, last_double_smoothed.y); + w = finish_double_smoothing(last_smoothed.w, last_double_smoothed.w); + h = finish_double_smoothing(last_smoothed.h, last_double_smoothed.h); + + // eventuell kein finish und stattdessen lastSmoothedValues verwenden (kein doublen) + // je nach performance kann ein Teil zuvor als Mean berechnet werden und als "kopfelement" + // in der exponentiellen Glättung verwendet werden. Aber das müssen wir testen. + + return bounding_box {int(x), int(y), int(w), int(h)}; + } }; #endif diff --git a/example/main.cpp b/example/main.cpp index 2eed72439..eea07691f 100644 --- a/example/main.cpp +++ b/example/main.cpp @@ -10,7 +10,7 @@ static void Help() { printf("\nExamples of the Multitarget tracking algorithm\n" "Usage: \n" - " ./MultitargetTracker [--example]= [--start_frame]= [--end_frame]= [--end_delay]= [--out]= [--show_logs]= [--async]= \n\n" + " ./MultitargetTracker [--example]= [--start_frame]= [--end_frame]= [--end_delay]= [--out]= [--show_logs]= [--async]= \n\n" "Press:\n" "\'m\' key for change mode: play|pause. When video is paused you can press any key for get next frame. \n\n" "Press Esc to exit from video \n\n" @@ -20,7 +20,7 @@ static void Help() const char* keys = { "{ @1 |../data/atrium.avi | movie file | }" - "{ e example |1 | number of example 0 - MouseTracking, 1 - MotionDetector, 2 - FaceDetector, 3 - PedestrianDetector, 4 - MobileNet SSD detector, 5 - Yolo OpenCV detector, 6 - Yolo Darknet detector | }" + "{ e example |1 | number of example 0 - MouseTracking, 1 - MotionDetector, 2 - FaceDetector, 3 - PedestrianDetector, 4 - MobileNet SSD detector, 5 - Yolo OpenCV detector, 6 - Yolo Darknet detector | 7 - SmartCamOperator w/ YoloDarknet | }" "{ sf start_frame |0 | Start a video from this position | }" "{ ef end_frame |0 | Play a video to this position (if 0 then played to the end of file) | }" "{ ed end_delay |0 | Delay in milliseconds after video ending | }" @@ -93,6 +93,13 @@ int main(int argc, char** argv) asyncPipeline ? yolo_detector.AsyncProcess() : yolo_detector.SyncProcess(); break; } + + case 7: + { + SmartCamOperator_w_YoloDarknet smart_cam_operator(parser); + asyncPipeline ? smart_cam_operator.AsyncProcess_SmartCamOp() : smart_cam_operator.SyncProcess(); + break; + } #endif default: diff --git a/src/Detector/YoloDarknetDetector.cpp b/src/Detector/YoloDarknetDetector.cpp index e1317e5cf..c607013f7 100644 --- a/src/Detector/YoloDarknetDetector.cpp +++ b/src/Detector/YoloDarknetDetector.cpp @@ -1,3 +1,4 @@ +#include #include "YoloDarknetDetector.h" #include "nms.h" diff --git a/src/Tracker/track.cpp b/src/Tracker/track.cpp index 1cacc7c4b..5ca6424bb 100644 --- a/src/Tracker/track.cpp +++ b/src/Tracker/track.cpp @@ -120,7 +120,7 @@ track_t CTrack::CalcDistHist(const CRegion& reg, cv::UMat currFrame) const } if (!reg.m_hist.empty() && !m_lastRegion.m_hist.empty()) { -#if (((CV_VERSION_MAJOR == 4) && (CV_VERSION_MINOR < 2)) || (CV_VERSION_MAJOR == 3)) +#if (((CV_VERSION_MAJOR == 4) && (CV_VERSION_MINOR < 1)) || (CV_VERSION_MAJOR == 3)) res = static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_BHATTACHARYYA)); //res = 1.f - static_cast(cv::compareHist(reg.m_hist, m_lastRegion.m_hist, CV_COMP_CORREL)); #else