From 589dc0f1c85f6e14a2d6a66e2c3669909ac10184 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Mon, 6 Dec 2021 12:36:28 +0900 Subject: [PATCH 01/19] dock: Implement empty dock Code is copied from obs-color-monitor. --- CMakeLists.txt | 19 +- ci/linux/install-dependencies-ubuntu.sh | 3 +- src/module-main.c | 9 + src/plugin-macros.h.in | 1 + ui/face-tracker-dock-internal.hpp | 33 +++ ui/face-tracker-dock.cpp | 185 +++++++++++++++ ui/face-tracker-dock.hpp | 43 ++++ ui/face-tracker-widget.cpp | 297 ++++++++++++++++++++++++ ui/face-tracker-widget.hpp | 55 +++++ ui/obsgui-helper.hpp | 46 ++++ 10 files changed, 689 insertions(+), 2 deletions(-) create mode 100644 ui/face-tracker-dock-internal.hpp create mode 100644 ui/face-tracker-dock.cpp create mode 100644 ui/face-tracker-dock.hpp create mode 100644 ui/face-tracker-widget.cpp create mode 100644 ui/face-tracker-widget.hpp create mode 100644 ui/obsgui-helper.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c5cc39..4ad4b7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ set(LINUX_MAINTAINER_EMAIL "norihiro@nagater.net") option(WITH_PTZ_TCP "Enable to connect PTZ camera through TCP socket" ON) option(ENABLE_MONITOR_USER "Enable monitor source for user" OFF) +option(WITH_DOCK "Enable dock" ON) set(CMAKE_PREFIX_PATH "${QTDIR}") set(CMAKE_AUTOMOC ON) @@ -21,7 +22,7 @@ include(external/FindLibObs.cmake) endif() find_package(LibObs REQUIRED) -find_package(Qt5 REQUIRED COMPONENTS Core Widgets) +find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui) set(plugin_additional_libs) if (WITH_PTZ_TCP) @@ -101,6 +102,22 @@ if (WITH_PTZ_TCP) set(PLUGIN_HEADERS ${PLUGIN_HEADERS} src/libvisca-thread.hpp) endif() +if (WITH_DOCK) + set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + ui/face-tracker-dock.cpp + ui/face-tracker-widget.cpp + ) + set(PLUGIN_HEADERS + ${PLUGIN_HEADERS} + ui/face-tracker-dock.hpp + ui/face-tracker-widget.hpp + ) + set(plugin_additional_incs ${plugin_additional_incs} src) + set(plugin_additional_libs ${plugin_additional_libs} Qt5::Gui Qt5::GuiPrivate) + set(plugin_additional_incs ${plugin_additional_incs} ${Qt5Gui_PRIVATE_INCLUDE_DIRS}) +endif() + # --- Platform-independent build settings --- add_library(${CMAKE_PROJECT_NAME} MODULE ${PLUGIN_SOURCES} ${PLUGIN_HEADERS}) diff --git a/ci/linux/install-dependencies-ubuntu.sh b/ci/linux/install-dependencies-ubuntu.sh index fd5a987..0588c26 100755 --- a/ci/linux/install-dependencies-ubuntu.sh +++ b/ci/linux/install-dependencies-ubuntu.sh @@ -11,7 +11,8 @@ sudo apt-get install -y \ checkinstall \ cmake \ obs-studio \ - qtbase5-dev + qtbase5-dev \ + qtbase5-private-dev # Dirty hack obs_version="$(dpkg -s obs-studio | awk '$1=="Version:" {print gensub(/-.*/, "", 1, $2)}')" diff --git a/src/module-main.c b/src/module-main.c index c379aa1..981cef4 100644 --- a/src/module-main.c +++ b/src/module-main.c @@ -1,5 +1,8 @@ #include #include "plugin-macros.generated.h" +#ifdef WITH_DOCK +#include "../ui/face-tracker-dock.hpp" +#endif // WITH_DOCK OBS_DECLARE_MODULE() OBS_MODULE_USE_DEFAULT_LOCALE(PLUGIN_NAME, "en-US") @@ -14,9 +17,15 @@ bool obs_module_load(void) register_face_tracker_filter(); register_face_tracker_ptz(); register_face_tracker_monitor(); +#ifdef WITH_DOCK + ft_docks_init(); +#endif // WITH_DOCK return true; } void obs_module_unload() { +#ifdef WITH_DOCK + ft_docks_release(); +#endif // WITH_DOCK } diff --git a/src/plugin-macros.h.in b/src/plugin-macros.h.in index d900f8f..e3ba3d0 100644 --- a/src/plugin-macros.h.in +++ b/src/plugin-macros.h.in @@ -24,6 +24,7 @@ with this program. If not, see #cmakedefine WITH_PTZ_TCP #cmakedefine ENABLE_MONITOR_USER +#cmakedefine WITH_DOCK #define blog(level, msg, ...) blog(level, "[" PLUGIN_NAME "] " msg, ##__VA_ARGS__) diff --git a/ui/face-tracker-dock-internal.hpp b/ui/face-tracker-dock-internal.hpp new file mode 100644 index 0000000..ba67d25 --- /dev/null +++ b/ui/face-tracker-dock-internal.hpp @@ -0,0 +1,33 @@ +#pragma once +#include +#include +#include "util/threading.h" + +struct face_tracker_dock_s +{ + obs_display_t *disp; + pthread_mutex_t mutex; + volatile long ref; +}; + +static inline struct face_tracker_dock_s *face_tracker_dock_create() +{ + struct face_tracker_dock_s *data = (struct face_tracker_dock_s *)bzalloc(sizeof(struct face_tracker_dock_s)); + data->ref = 1; + return data; +} + +static inline void face_tracker_dock_addref(struct face_tracker_dock_s *data) +{ + if (!data) + return; + os_atomic_inc_long(&data->ref); +} + +static inline void face_tracker_dock_release(struct face_tracker_dock_s *data) +{ + if (!data) + return; + if (os_atomic_dec_long(&data->ref) == 0) + bfree(data); +} diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp new file mode 100644 index 0000000..bd31aab --- /dev/null +++ b/ui/face-tracker-dock.cpp @@ -0,0 +1,185 @@ +#include +#include +#include +#include +#include "plugin-macros.generated.h" +#include "face-tracker-dock.hpp" +#include "face-tracker-widget.hpp" +#include "face-tracker-dock-internal.hpp" + +#define SAVE_DATA_NAME PLUGIN_NAME"-dock" +#define OBJ_NAME_SUFFIX "_ft_dock" + +void FTDock::closeEvent(QCloseEvent *event) +{ + QDockWidget::closeEvent(event); +} + +// accessed only from UI thread +static std::vector *docks; + +static std::string generate_unique_name() +{ + for (int n=0;;) { + char name[32] = "FTDock"; + if (n) + snprintf(name, sizeof(name), "FTDock-%d", n); + bool found = false; + if (docks) for (size_t i=0; isize(); i++) { + if ((*docks)[i]->name == name) + found = true; + } + if (!found) + return name; + } +} + +void ft_dock_add(const char *name, obs_data_t *props) +{ + auto *main_window = static_cast(obs_frontend_get_main_window()); + auto *dock = new FTDock(main_window); + dock->name = name ? name : generate_unique_name(); + dock->setObjectName(QString::fromUtf8(dock->name.c_str()) + OBJ_NAME_SUFFIX); + dock->setWindowTitle(dock->name.c_str()); + dock->resize(256, 256); + dock->setMinimumSize(128, 128); + dock->setAllowedAreas(Qt::AllDockWidgetAreas); + + FTWidget *w = new FTWidget(dock); + dock->SetWidget(w); + dock->load_properties(props); + + main_window->addDockWidget(Qt::BottomDockWidgetArea, dock); + dock->action = (QAction*)obs_frontend_add_dock(dock); + + if (docks) + docks->push_back(dock); +} + +FTDock::FTDock(QWidget *parent) + : QDockWidget(parent) +{ + data = face_tracker_dock_create(); + setAttribute(Qt::WA_DeleteOnClose); +} + +FTDock::~FTDock() +{ + face_tracker_dock_release(data); + if (action) + delete action; + if (docks) for (size_t i=0; isize(); i++) { + if ((*docks)[i] == this) { + docks->erase(docks->begin()+i); + break; + } + } +} + +void FTDock::SetWidget(class FTWidget *w) +{ + w->SetData(data); + widget = w; + setWidget((QWidget*)w); +} + +void FTDock::showEvent(QShowEvent *event) +{ + blog(LOG_INFO, "FTDock::showEvent"); + widget->setShown(true); +} + +void FTDock::hideEvent(QHideEvent *event) +{ + blog(LOG_INFO, "FTDock::hideEvent"); + widget->setShown(false); +} + +static void save_load_ft_docks(obs_data_t *save_data, bool saving, void *) +{ + blog(LOG_INFO, "save_load_ft_docks saving=%d", (int)saving); + if (!docks) + return; + if (saving) { + obs_data_t *props = obs_data_create(); + obs_data_array_t *array = obs_data_array_create(); + for (size_t i=0; isize(); i++) { + FTDock *d = (*docks)[i]; + obs_data_t *obj = obs_data_create(); + d->save_properties(obj); + obs_data_set_string(obj, "name", d->name.c_str()); + obs_data_array_push_back(array, obj); + obs_data_release(obj); + } + obs_data_set_array(props, "docks", array); + obs_data_set_obj(save_data, SAVE_DATA_NAME, props); + obs_data_array_release(array); + obs_data_release(props); + } + + else /* loading */ { + if (docks) while (docks->size()) { + (*docks)[docks->size()-1]->close(); + delete (*docks)[docks->size()-1]; + } + + obs_data_t *props = obs_data_get_obj(save_data, SAVE_DATA_NAME); + if (!props) { + blog(LOG_INFO, "save_load_ft_docks: creating default properties"); + props = obs_data_create(); + } + + obs_data_array_t *array = obs_data_get_array(props, "docks"); + size_t count = obs_data_array_count(array); + for (size_t i=0; i; + obs_frontend_add_save_callback(save_load_ft_docks, NULL); + + QAction *action = static_cast(obs_frontend_add_tools_menu_qaction( + obs_module_text("New Face Tracker Dock...") )); + blog(LOG_INFO, "ft_docks_init: Adding face tracker dock menu action=%p", action); + auto cb = [] { + obs_data_t *props = obs_data_create(); + FTDock::default_properties(props); + ft_dock_add(NULL, props); + obs_data_release(props); + }; + QAction::connect(action, &QAction::triggered, cb); +} + +void ft_docks_release() +{ + delete docks; + docks = NULL; +} + +void FTDock::default_properties(obs_data_t *props) +{ +} + +void FTDock::save_properties(obs_data_t *props) +{ + // pthread_mutex_lock(&data->mutex); + // TODO: implement me + // pthread_mutex_unlock(&data->mutex); +} + +void FTDock::load_properties(obs_data_t *props) +{ + // pthread_mutex_lock(&data->mutex); + // TODO: implement me + // pthread_mutex_unlock(&data->mutex); +} diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp new file mode 100644 index 0000000..3e3add3 --- /dev/null +++ b/ui/face-tracker-dock.hpp @@ -0,0 +1,43 @@ +#pragma once + +#ifdef __cplusplus +#include +#include +#include +#include +#include + +class FTDock : public QDockWidget { + Q_OBJECT + +public: + class FTWidget *widget; + std::string name; + QPointer action = 0; + struct face_tracker_dock_s *data; + +public: + FTDock(QWidget *parent = nullptr); + ~FTDock(); + void closeEvent(QCloseEvent *event) override; + + static void default_properties(obs_data_t *); + void save_properties(obs_data_t*); + void load_properties(obs_data_t*); + + void SetWidget(class FTWidget *w); + +private: + void showEvent(QShowEvent *event) override; + void hideEvent(QHideEvent *event) override; +}; + +extern "C" { +#endif // __cplusplus +void ft_dock_add(const char *name, obs_data_t *props); +void ft_docks_init(); +void ft_docks_release(); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/ui/face-tracker-widget.cpp b/ui/face-tracker-widget.cpp new file mode 100644 index 0000000..1c0b7ea --- /dev/null +++ b/ui/face-tracker-widget.cpp @@ -0,0 +1,297 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "plugin-macros.generated.h" +#include "face-tracker-widget.hpp" +#include "face-tracker-dock-internal.hpp" +#include "obsgui-helper.hpp" + + +static void draw(void *param, uint32_t cx, uint32_t cy) +{ + auto *data = (struct face_tracker_dock_s *)param; + + + if (pthread_mutex_trylock(&data->mutex)) + return; + + //const auto src_shown = data->src_shown; + + gs_blend_state_push(); + gs_reset_blend_state(); + + gs_blend_state_pop(); + + pthread_mutex_unlock(&data->mutex); +} + +FTWidget::FTWidget(QWidget *parent) + : QWidget(parent) + , eventFilter(BuildEventFilter()) +{ + data = NULL; + setAttribute(Qt::WA_PaintOnScreen); + setAttribute(Qt::WA_StaticContents); + setAttribute(Qt::WA_NoSystemBackground); + setAttribute(Qt::WA_OpaquePaintEvent); + setAttribute(Qt::WA_DontCreateNativeAncestors); + setAttribute(Qt::WA_NativeWindow); + + setMouseTracking(true); + QObject::installEventFilter(eventFilter.get()); +} + +FTWidget::~FTWidget() +{ + removeEventFilter(eventFilter.get()); + face_tracker_dock_release(data); +} + +void FTWidget::SetData(struct face_tracker_dock_s *d) +{ + face_tracker_dock_addref(d); + face_tracker_dock_release(data); + data = d; +} + +OBSEventFilter *FTWidget::BuildEventFilter() +{ + return new OBSEventFilter([this](QObject *obj, QEvent *event) { + UNUSED_PARAMETER(obj); + + switch (event->type()) { + case QEvent::MouseButtonPress: + case QEvent::MouseButtonRelease: + case QEvent::MouseButtonDblClick: + return this->HandleMouseClickEvent( + static_cast(event)); + case QEvent::MouseMove: + case QEvent::Enter: + case QEvent::Leave: + return this->HandleMouseMoveEvent( + static_cast(event)); + + case QEvent::Wheel: + return this->HandleMouseWheelEvent( + static_cast(event)); + case QEvent::KeyPress: + case QEvent::KeyRelease: + return this->HandleKeyEvent( + static_cast(event)); + default: + return false; + } + }); +} + +void FTWidget::CreateDisplay() +{ + if (data->disp || !windowHandle()->isExposed()) + return; + + blog(LOG_INFO, "FTWidget::CreateDisplay %p", this); + + QSize size = GetPixelSize(this); + gs_init_data info = {}; + info.cx = size.width(); + info.cy = size.height(); + info.format = GS_BGRA; + info.zsformat = GS_ZS_NONE; + QWindow *window = windowHandle(); + if (!window) { + blog(LOG_ERROR, "FTWidget %p: windowHandle() returns NULL", this); + return; + } + if (!QTToGSWindow(window, info.window)) { + blog(LOG_ERROR, "FTWidget %p: QTToGSWindow failed", this); + return; + } + data->disp = obs_display_create(&info, 0); + obs_display_add_draw_callback(data->disp, draw, data); +} + +void FTWidget::resizeEvent(QResizeEvent *event) +{ + QWidget::resizeEvent(event); + CreateDisplay(); + + QSize size = GetPixelSize(this); + obs_display_resize(data->disp, size.width(), size.height()); +} + +void FTWidget::paintEvent(QPaintEvent *event) +{ + CreateDisplay(); +} + +class QPaintEngine *FTWidget::paintEngine() const +{ + return NULL; +} + +void FTWidget::closeEvent(QCloseEvent *event) +{ + setShown(false); +} + +void FTWidget::setShown(bool shown) +{ + if (shown && !data->disp) { + CreateDisplay(); + } + if (!shown && data->disp) { + obs_display_destroy(data->disp); + data->disp = NULL; + } +} + +#define INTERACT_KEEP_SOURCE (1<<30) + +static int TranslateQtKeyboardEventModifiers(QInputEvent *event, + bool mouseEvent) +{ + int obsModifiers = INTERACT_NONE; + + if (event->modifiers().testFlag(Qt::ShiftModifier)) + obsModifiers |= INTERACT_SHIFT_KEY; + if (event->modifiers().testFlag(Qt::AltModifier)) + obsModifiers |= INTERACT_ALT_KEY; +#ifdef __APPLE__ + // Mac: Meta = Control, Control = Command + if (event->modifiers().testFlag(Qt::ControlModifier)) + obsModifiers |= INTERACT_COMMAND_KEY; + if (event->modifiers().testFlag(Qt::MetaModifier)) + obsModifiers |= INTERACT_CONTROL_KEY; +#else + // Handle windows key? Can a browser even trap that key? + if (event->modifiers().testFlag(Qt::ControlModifier)) + obsModifiers |= INTERACT_CONTROL_KEY; +#endif + + if (!mouseEvent) { + if (event->modifiers().testFlag(Qt::KeypadModifier)) + obsModifiers |= INTERACT_IS_KEY_PAD; + } + + return obsModifiers; +} + +static int TranslateQtMouseEventModifiers(QMouseEvent *event) +{ + int modifiers = TranslateQtKeyboardEventModifiers(event, true); + + if (event->buttons().testFlag(Qt::LeftButton)) + modifiers |= INTERACT_MOUSE_LEFT; + if (event->buttons().testFlag(Qt::MiddleButton)) + modifiers |= INTERACT_MOUSE_MIDDLE; + if (event->buttons().testFlag(Qt::RightButton)) + modifiers |= INTERACT_MOUSE_RIGHT; + + return modifiers; +} + +bool FTWidget::HandleMouseClickEvent(QMouseEvent *event) +{ + bool mouseUp = event->type() == QEvent::MouseButtonRelease; + int clickCount = 1; + if (event->type() == QEvent::MouseButtonDblClick) + clickCount = 2; + + struct obs_mouse_event mouseEvent = {}; + + mouseEvent.modifiers = TranslateQtMouseEventModifiers(event); + + int32_t button = 0; + + switch (event->button()) { + case Qt::LeftButton: + button = MOUSE_LEFT; + if (mouseUp) mouseEvent.modifiers |= INTERACT_KEEP_SOURCE; // Not to change i_mouse if released outside + break; + case Qt::MiddleButton: + button = MOUSE_MIDDLE; + break; + case Qt::RightButton: + button = MOUSE_RIGHT; + break; + default: + blog(LOG_WARNING, "unknown button type %d", event->button()); + return false; + } + + return true; +} + +bool FTWidget::HandleMouseMoveEvent(QMouseEvent *event) +{ + struct obs_mouse_event mouseEvent = {}; + + bool mouseLeave = event->type() == QEvent::Leave; + + if (!mouseLeave) + mouseEvent.modifiers = TranslateQtMouseEventModifiers(event); + + int x = event->x(); + int y = event->y(); + + return true; +} + +bool FTWidget::HandleMouseWheelEvent(QWheelEvent *event) +{ + struct obs_mouse_event mouseEvent = {}; + + mouseEvent.modifiers = TranslateQtKeyboardEventModifiers(event, true); + + int xDelta = 0; + int yDelta = 0; + + const QPoint angleDelta = event->angleDelta(); + if (!event->pixelDelta().isNull()) { + if (angleDelta.x()) + xDelta = event->pixelDelta().x(); + else + yDelta = event->pixelDelta().y(); + } else { + if (angleDelta.x()) + xDelta = angleDelta.x(); + else + yDelta = angleDelta.y(); + } + +#if QT_VERSION >= QT_VERSION_CHECK(5, 14, 0) + const QPointF position = event->position(); + const int x = position.x(); + const int y = position.y(); +#else + const int x = event->x(); + const int y = event->y(); +#endif + + return true; +} + +bool FTWidget::HandleKeyEvent(QKeyEvent *event) +{ + struct obs_key_event keyEvent; + + QByteArray text = event->text().toUtf8(); + keyEvent.modifiers = TranslateQtKeyboardEventModifiers(event, false); + keyEvent.text = text.data(); + keyEvent.native_modifiers = event->nativeModifiers(); + keyEvent.native_scancode = event->nativeScanCode(); + keyEvent.native_vkey = event->nativeVirtualKey(); + + bool keyUp = event->type() == QEvent::KeyRelease; + + // TODO: implement me obs_source_send_key_click(source, &keyEvent, keyUp); + + return true; +} diff --git a/ui/face-tracker-widget.hpp b/ui/face-tracker-widget.hpp new file mode 100644 index 0000000..8a76d50 --- /dev/null +++ b/ui/face-tracker-widget.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +#define SCOPE_WIDGET_N_SRC 4 + +class OBSEventFilter; + +class FTWidget : public QWidget { + Q_OBJECT + + std::unique_ptr eventFilter; + + void CreateDisplay(); + void resizeEvent(QResizeEvent *event) override; + void paintEvent(QPaintEvent *event) override; + class QPaintEngine *paintEngine() const override; + void closeEvent(QCloseEvent *event) override; + + // for interactions + bool HandleMouseClickEvent(QMouseEvent *event); + bool HandleMouseMoveEvent(QMouseEvent *event); + bool HandleMouseWheelEvent(QWheelEvent *event); + bool HandleKeyEvent(QKeyEvent *event); + OBSEventFilter *BuildEventFilter(); + +public: + FTWidget(QWidget *parent); + ~FTWidget(); + void SetData(struct face_tracker_dock_s *data); + void setShown(bool shown); + +private: + struct face_tracker_dock_s *data; +}; + +typedef std::function EventFilterFunc; + +class OBSEventFilter : public QObject +{ + Q_OBJECT +public: + OBSEventFilter(EventFilterFunc filter_) : filter(filter_) {} + +protected: + bool eventFilter(QObject *obj, QEvent *event) + { + return filter(obj, event); + } + +public: + EventFilterFunc filter; +}; diff --git a/ui/obsgui-helper.hpp b/ui/obsgui-helper.hpp new file mode 100644 index 0000000..c691a6c --- /dev/null +++ b/ui/obsgui-helper.hpp @@ -0,0 +1,46 @@ +#pragma once +#include +#if !defined(_WIN32) && !defined(__APPLE__) // if Linux +#include +#include +#include +#endif + +// copied from obs-studio/UI/qt-wrappers.cpp and modified to support OBS-26 +static inline +bool QTToGSWindow(QWindow *window, gs_window &gswindow) +{ + bool success = true; + +#ifdef _WIN32 + gswindow.hwnd = (HWND)window->winId(); +#elif __APPLE__ + gswindow.view = (id)window->winId(); +#else +#ifdef ENABLE_WAYLAND + switch (obs_get_nix_platform()) { + case OBS_NIX_PLATFORM_X11_GLX: + case OBS_NIX_PLATFORM_X11_EGL: +#endif // ENABLE_WAYLAND + gswindow.id = window->winId(); + gswindow.display = obs_get_nix_platform_display(); +#ifdef ENABLE_WAYLAND + break; + case OBS_NIX_PLATFORM_WAYLAND: + QPlatformNativeInterface *native = + QGuiApplication::platformNativeInterface(); + gswindow.display = + native->nativeResourceForWindow("surface", window); + success = gswindow.display != nullptr; + break; + } +#endif // ENABLE_WAYLAND +#endif + return success; +} + +// copied from obs-studio/UI/display-helpers.hpp +static inline QSize GetPixelSize(QWidget *widget) +{ + return widget->size() * widget->devicePixelRatioF(); +} From 9d1e7afd0eb316bc79c36b5f511f1971df9b71d3 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Tue, 7 Dec 2021 23:08:10 +0900 Subject: [PATCH 02/19] doc: Deprecate preset from the properties The preset feature will be provided through a dock. Preset feature will be removed from the property dialog in the future release. --- doc/properties-ptz.md | 2 ++ doc/properties.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/doc/properties-ptz.md b/doc/properties-ptz.md index 6719633..3d17fb4 100644 --- a/doc/properties-ptz.md +++ b/doc/properties-ptz.md @@ -5,6 +5,8 @@ When clicked, tracking state is reset to the initial condition; reset internal s (This is not a property.) ## Preset +**Deprecated** +Preset will be provided through a dock. The preset group in the property dialog will be removed in the future release. ### Preset This combo-box sets the name of the preset to be loaded or saved. diff --git a/doc/properties.md b/doc/properties.md index 0944b07..7f5a443 100644 --- a/doc/properties.md +++ b/doc/properties.md @@ -5,6 +5,8 @@ When clicked, tracking state is reset to the initial condition; zero crop, reset (This is not a property.) ## Preset +**Deprecated** +Preset will be provided through a dock. The preset group in the property dialog will be removed in the future release. ### Preset This combo-box sets the name of the preset to be loaded or saved. From 9d9b95fa64107b520dca134af83d9968b252ef42 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 25 Dec 2021 21:09:27 +0900 Subject: [PATCH 03/19] dock: Implement source/filter selector --- ui/face-tracker-dock.cpp | 120 ++++++++++++++++++++++++++++++++++++--- ui/face-tracker-dock.hpp | 12 ++++ 2 files changed, 125 insertions(+), 7 deletions(-) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index bd31aab..6872ade 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include "plugin-macros.generated.h" #include "face-tracker-dock.hpp" #include "face-tracker-widget.hpp" @@ -41,12 +43,8 @@ void ft_dock_add(const char *name, obs_data_t *props) dock->name = name ? name : generate_unique_name(); dock->setObjectName(QString::fromUtf8(dock->name.c_str()) + OBJ_NAME_SUFFIX); dock->setWindowTitle(dock->name.c_str()); - dock->resize(256, 256); - dock->setMinimumSize(128, 128); dock->setAllowedAreas(Qt::AllDockWidgetAreas); - FTWidget *w = new FTWidget(dock); - dock->SetWidget(w); dock->load_properties(props); main_window->addDockWidget(Qt::BottomDockWidgetArea, dock); @@ -56,15 +54,119 @@ void ft_dock_add(const char *name, obs_data_t *props) docks->push_back(dock); } +struct init_target_selector_s +{ + QComboBox *q; + int index; +}; + +void init_target_selector_cb_add(struct init_target_selector_s *ctx, obs_source_t *source, obs_source_t *filter) +{ + QString text; + QList val; + + const char *name = obs_source_get_name(source); + text = QString::fromUtf8(name); + val.append(QVariant(name)); + + if (filter) { + const char *name = obs_source_get_name(filter); + text += " / "; + text += QString::fromUtf8(name); + val.append(QVariant(name)); + } + + if (ctx->index < ctx->q->count()) { + ctx->q->setItemText(ctx->index, text); + ctx->q->setItemData(ctx->index, QVariant(val)); + } + else + ctx->q->insertItem(ctx->index, text, QVariant(val)); + ctx->index++; +} + +static void init_target_selector_cb_filter(obs_source_t *parent, obs_source_t *child, void *param) +{ + auto *ctx = (struct init_target_selector_s *)param; + + const char *id = obs_source_get_id(child); + if (!strcmp(id, "face_tracker_filter") || !strcmp(id, "face_tracker_ptz")) { + init_target_selector_cb_add(ctx, parent, child); + } +} + +static bool init_target_selector_cb_source(void *data, obs_source_t *source) +{ + auto *ctx = (struct init_target_selector_s *)data; + + const char *id = obs_source_get_id(source); + if (!strcmp(id, "face_tracker_source")) { + init_target_selector_cb_add(ctx, source, NULL); + return true; + } + + obs_source_enum_filters(source, init_target_selector_cb_filter, data); + + return true; +} + +static void init_target_selector(QComboBox *q) +{ + QString current = q->currentText(); + + init_target_selector_s ctx = {q, 0}; + obs_enum_scenes(init_target_selector_cb_source, &ctx); + obs_enum_sources(init_target_selector_cb_source, &ctx); + + while (q->count() > ctx.index) + q->removeItem(ctx.index); + + if (current.length()) { + int ix = q->findText(current); + if (ix >= 0) + q->setCurrentIndex(ix); + } +} + +void FTDock::checkTargetSelector() +{ + init_target_selector(targetSelector); +} + +void FTDock::frontendEvent_cb(enum obs_frontend_event event, void *private_data) +{ + auto *dock = static_cast(private_data); + dock->frontendEvent(event); +} + FTDock::FTDock(QWidget *parent) : QDockWidget(parent) { data = face_tracker_dock_create(); + + resize(256, 256); + setMinimumSize(128, 128); setAttribute(Qt::WA_DeleteOnClose); + + mainLayout = new QVBoxLayout(this); + auto *dockWidgetContents = new QWidget; + dockWidgetContents->setObjectName(QStringLiteral("contextContainer")); + dockWidgetContents->setLayout(mainLayout); + + targetSelector = new QComboBox(this); + init_target_selector(targetSelector); + QObject::connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); + mainLayout->addWidget(targetSelector); + + setWidget(dockWidgetContents); + + obs_frontend_add_event_callback(frontendEvent_cb, this); } FTDock::~FTDock() { + obs_frontend_remove_event_callback(frontendEvent_cb, this); + face_tracker_dock_release(data); if (action) delete action; @@ -80,19 +182,20 @@ void FTDock::SetWidget(class FTWidget *w) { w->SetData(data); widget = w; - setWidget((QWidget*)w); } void FTDock::showEvent(QShowEvent *event) { blog(LOG_INFO, "FTDock::showEvent"); - widget->setShown(true); } void FTDock::hideEvent(QHideEvent *event) { blog(LOG_INFO, "FTDock::hideEvent"); - widget->setShown(false); +} + +void FTDock::frontendEvent(enum obs_frontend_event event) +{ } static void save_load_ft_docks(obs_data_t *save_data, bool saving, void *) @@ -172,6 +275,9 @@ void FTDock::default_properties(obs_data_t *props) void FTDock::save_properties(obs_data_t *props) { + // Save indicates a source or a filter has been changed. + scenesMayChanged(); + // pthread_mutex_lock(&data->mutex); // TODO: implement me // pthread_mutex_unlock(&data->mutex); diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index 3e3add3..dc4f2b8 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -6,6 +6,7 @@ #include #include #include +#include class FTDock : public QDockWidget { Q_OBJECT @@ -16,6 +17,9 @@ class FTDock : public QDockWidget { QPointer action = 0; struct face_tracker_dock_s *data; + class QVBoxLayout *mainLayout; + class QComboBox *targetSelector; + public: FTDock(QWidget *parent = nullptr); ~FTDock(); @@ -30,6 +34,14 @@ class FTDock : public QDockWidget { private: void showEvent(QShowEvent *event) override; void hideEvent(QHideEvent *event) override; + + void frontendEvent(enum obs_frontend_event event); + static void frontendEvent_cb(enum obs_frontend_event event, void *private_data); + +signals: + void scenesMayChanged(); +public slots: + void checkTargetSelector(); }; extern "C" { From 6ae5bf6308fac32b5a52ae06ebb0b8c76feaa493 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 25 Dec 2021 23:52:18 +0900 Subject: [PATCH 04/19] Add get_state and set_state procedures To control from dock, add get_state and set_state procedure for source and filters. Currently, only `paused` is available but might add more functions in future. --- src/face-tracker-ptz.cpp | 16 ++++++++++++++++ src/face-tracker.cpp | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/face-tracker-ptz.cpp b/src/face-tracker-ptz.cpp index a22f23c..456260b 100644 --- a/src/face-tracker-ptz.cpp +++ b/src/face-tracker-ptz.cpp @@ -246,6 +246,8 @@ static void ftptz_update(void *data, obs_data_t *settings) } static void cb_render_info(void *data, calldata_t *cd); +static void cb_get_state(void *data, calldata_t *cd); +static void cb_set_state(void *data, calldata_t *cd); static void *ftptz_create(obs_data_t *settings, obs_source_t *context) { @@ -261,6 +263,8 @@ static void *ftptz_create(obs_data_t *settings, obs_source_t *context) proc_handler_t *ph = obs_source_get_proc_handler(context); proc_handler_add(ph, "void render_info()", cb_render_info, s); + proc_handler_add(ph, "void get_state()", cb_get_state, s); + proc_handler_add(ph, "void set_state()", cb_set_state, s); return s; } @@ -849,6 +853,18 @@ static void cb_render_info(void *data, calldata_t *cd) draw_frame_info(s, landmark_only); } +static void cb_get_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_ptz*)data; + calldata_set_bool(cd, "paused", s->is_paused); +} + +static void cb_set_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_ptz*)data; + calldata_get_bool(cd, "paused", &s->is_paused); +} + extern "C" void register_face_tracker_ptz() { diff --git a/src/face-tracker.cpp b/src/face-tracker.cpp index b8fe66d..30f5e88 100644 --- a/src/face-tracker.cpp +++ b/src/face-tracker.cpp @@ -116,6 +116,8 @@ static void fts_update(void *data, obs_data_t *settings) static void cb_render_frame(void *data, calldata_t *cd); static void cb_render_info(void *data, calldata_t *cd); static void cb_get_target_size(void *data, calldata_t *cd); +static void cb_get_state(void *data, calldata_t *cd); +static void cb_set_state(void *data, calldata_t *cd); static void *ftf_create(obs_data_t *settings, obs_source_t *context) { @@ -143,6 +145,8 @@ static void *ftf_create(obs_data_t *settings, obs_source_t *context) proc_handler_add(ph, "void render_frame(bool notrack)", cb_render_frame, s); proc_handler_add(ph, "void render_info(bool notrack)", cb_render_info, s); proc_handler_add(ph, "void get_target_size(out int width, out int height)", cb_get_target_size, s); + proc_handler_add(ph, "void get_state()", cb_get_state, s); + proc_handler_add(ph, "void set_state()", cb_set_state, s); return s; } @@ -931,6 +935,18 @@ static void cb_get_target_size(void *data, calldata_t *cd) calldata_set_int(cd, "height", (int)s->known_height); } +static void cb_get_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_filter*)data; + calldata_set_bool(cd, "paused", s->is_paused); +} + +static void cb_set_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_filter*)data; + calldata_get_bool(cd, "paused", &s->is_paused); +} + extern "C" void register_face_tracker_filter() { From 32994e3ebcf447a6e2178ca30d51520e700e2502 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 25 Dec 2021 23:54:19 +0900 Subject: [PATCH 05/19] dock: Add pause control --- ui/face-tracker-dock.cpp | 63 +++++++++++++++++++++++++++++++++++++++- ui/face-tracker-dock.hpp | 10 +++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index 6872ade..571f4c7 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "plugin-macros.generated.h" #include "face-tracker-dock.hpp" #include "face-tracker-widget.hpp" @@ -155,11 +156,18 @@ FTDock::FTDock(QWidget *parent) targetSelector = new QComboBox(this); init_target_selector(targetSelector); - QObject::connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); mainLayout->addWidget(targetSelector); + connect(targetSelector, &QComboBox::currentTextChanged, this, &FTDock::targetSelectorChanged); + + pauseButton = new QCheckBox(obs_module_text("Pause"), this); + mainLayout->addWidget(pauseButton); + connect(pauseButton, &QCheckBox::stateChanged, this, &FTDock::pauseButtonChanged); setWidget(dockWidgetContents); + connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); + updateState(); + obs_frontend_add_event_callback(frontendEvent_cb, this); } @@ -198,6 +206,59 @@ void FTDock::frontendEvent(enum obs_frontend_event event) { } +void FTDock::targetSelectorChanged() +{ + updateState(); +} + +OBSSource FTDock::get_source() +{ + OBSSource target; + QList data = targetSelector->currentData().toList(); + + for (int i=0; isetCheckState(b ? Qt::Checked : Qt::Unchecked); + } + } + calldata_free(&cd); +} + +void FTDock::pauseButtonChanged(int state) +{ + OBSSource target = get_source(); + proc_handler_t *ph = obs_source_get_proc_handler(target); + if (!ph) + return; + + calldata_t cd = {0}; + calldata_set_bool(&cd, "paused", state==Qt::Checked); + proc_handler_call(ph, "set_state", &cd); + calldata_free(&cd); +} + static void save_load_ft_docks(obs_data_t *save_data, bool saving, void *) { blog(LOG_INFO, "save_load_ft_docks saving=%d", (int)saving); diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index dc4f2b8..ce52524 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include class FTDock : public QDockWidget { @@ -19,6 +20,7 @@ class FTDock : public QDockWidget { class QVBoxLayout *mainLayout; class QComboBox *targetSelector; + class QCheckBox *pauseButton; public: FTDock(QWidget *parent = nullptr); @@ -38,10 +40,18 @@ class FTDock : public QDockWidget { void frontendEvent(enum obs_frontend_event event); static void frontendEvent_cb(enum obs_frontend_event event, void *private_data); + OBSSource get_source(); + signals: void scenesMayChanged(); + public slots: void checkTargetSelector(); + void updateState(); + +private slots: + void targetSelectorChanged(); + void pauseButtonChanged(int state); }; extern "C" { From 40374165f5167047edcf045a1708f085aea378b8 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sun, 26 Dec 2021 00:46:19 +0900 Subject: [PATCH 06/19] Add reset feature to set_state procedure --- src/face-tracker-ptz.cpp | 5 +++++ src/face-tracker.cpp | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/face-tracker-ptz.cpp b/src/face-tracker-ptz.cpp index 456260b..09d5ddb 100644 --- a/src/face-tracker-ptz.cpp +++ b/src/face-tracker-ptz.cpp @@ -863,6 +863,11 @@ static void cb_set_state(void *data, calldata_t *cd) { auto *s = (struct face_tracker_ptz*)data; calldata_get_bool(cd, "paused", &s->is_paused); + + bool reset = false; + calldata_get_bool(cd, "reset", &reset); + if (reset) + ftptz_reset_tracking(NULL, NULL, s); } extern "C" diff --git a/src/face-tracker.cpp b/src/face-tracker.cpp index 30f5e88..d9a07c5 100644 --- a/src/face-tracker.cpp +++ b/src/face-tracker.cpp @@ -945,6 +945,11 @@ static void cb_set_state(void *data, calldata_t *cd) { auto *s = (struct face_tracker_filter*)data; calldata_get_bool(cd, "paused", &s->is_paused); + + bool reset = false; + calldata_get_bool(cd, "reset", &reset); + if (reset) + ftf_reset_tracking(NULL, NULL, s); } extern "C" From de1f67641d18c3fc29ada62cf5a4e72197cbd8cb Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sun, 26 Dec 2021 00:47:30 +0900 Subject: [PATCH 07/19] dock: Add reset button --- ui/face-tracker-dock.cpp | 20 ++++++++++++++++++++ ui/face-tracker-dock.hpp | 2 ++ 2 files changed, 22 insertions(+) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index 571f4c7..a238417 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "plugin-macros.generated.h" #include "face-tracker-dock.hpp" #include "face-tracker-widget.hpp" @@ -163,6 +164,10 @@ FTDock::FTDock(QWidget *parent) mainLayout->addWidget(pauseButton); connect(pauseButton, &QCheckBox::stateChanged, this, &FTDock::pauseButtonChanged); + resetButton = new QPushButton(obs_module_text("Reset"), this); + mainLayout->addWidget(resetButton); + connect(resetButton, &QPushButton::clicked, this, &FTDock::resetButtonClicked); + setWidget(dockWidgetContents); connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); @@ -259,6 +264,21 @@ void FTDock::pauseButtonChanged(int state) calldata_free(&cd); } +void FTDock::resetButtonClicked(bool checked) +{ + UNUSED_PARAMETER(checked); + + OBSSource target = get_source(); + proc_handler_t *ph = obs_source_get_proc_handler(target); + if (!ph) + return; + + calldata_t cd = {0}; + calldata_set_bool(&cd, "reset", true); + proc_handler_call(ph, "set_state", &cd); + calldata_free(&cd); +} + static void save_load_ft_docks(obs_data_t *save_data, bool saving, void *) { blog(LOG_INFO, "save_load_ft_docks saving=%d", (int)saving); diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index ce52524..95131fa 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -21,6 +21,7 @@ class FTDock : public QDockWidget { class QVBoxLayout *mainLayout; class QComboBox *targetSelector; class QCheckBox *pauseButton; + class QPushButton *resetButton; public: FTDock(QWidget *parent = nullptr); @@ -52,6 +53,7 @@ public slots: private slots: void targetSelectorChanged(); void pauseButtonChanged(int state); + void resetButtonClicked(bool checked); }; extern "C" { From 39c5e0810a6734ef5982ac4dc372fee64837b9f8 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Tue, 28 Dec 2021 19:11:30 +0900 Subject: [PATCH 08/19] dock: Add monitor and render it --- ui/face-tracker-dock-internal.hpp | 6 +++- ui/face-tracker-dock.cpp | 47 +++++++++++++++++++++++++++---- ui/face-tracker-dock.hpp | 3 +- ui/face-tracker-widget.cpp | 43 ++++++++++++++++++++-------- ui/face-tracker-widget.hpp | 3 +- 5 files changed, 79 insertions(+), 23 deletions(-) diff --git a/ui/face-tracker-dock-internal.hpp b/ui/face-tracker-dock-internal.hpp index ba67d25..daca7c9 100644 --- a/ui/face-tracker-dock-internal.hpp +++ b/ui/face-tracker-dock-internal.hpp @@ -8,6 +8,8 @@ struct face_tracker_dock_s obs_display_t *disp; pthread_mutex_t mutex; volatile long ref; + + obs_source_t *src_monitor; }; static inline struct face_tracker_dock_s *face_tracker_dock_create() @@ -24,10 +26,12 @@ static inline void face_tracker_dock_addref(struct face_tracker_dock_s *data) os_atomic_inc_long(&data->ref); } +void face_tracker_dock_destroy(struct face_tracker_dock_s *data); + static inline void face_tracker_dock_release(struct face_tracker_dock_s *data) { if (!data) return; if (os_atomic_dec_long(&data->ref) == 0) - bfree(data); + face_tracker_dock_destroy(data); } diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index a238417..d02ddcb 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -146,6 +146,8 @@ FTDock::FTDock(QWidget *parent) { data = face_tracker_dock_create(); + data->src_monitor = obs_source_create_private("face_tracker_monitor", "monitor", NULL); + resize(256, 256); setMinimumSize(128, 128); setAttribute(Qt::WA_DeleteOnClose); @@ -168,6 +170,9 @@ FTDock::FTDock(QWidget *parent) mainLayout->addWidget(resetButton); connect(resetButton, &QPushButton::clicked, this, &FTDock::resetButtonClicked); + ftWidget = new FTWidget(data, this); + mainLayout->addWidget(ftWidget); + setWidget(dockWidgetContents); connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); @@ -191,12 +196,6 @@ FTDock::~FTDock() } } -void FTDock::SetWidget(class FTWidget *w) -{ - w->SetData(data); - widget = w; -} - void FTDock::showEvent(QShowEvent *event) { blog(LOG_INFO, "FTDock::showEvent"); @@ -233,6 +232,24 @@ OBSSource FTDock::get_source() return target; } +static inline void set_monitor(obs_source_t *monitor, const QList &target_data) +{ + blog(LOG_INFO, "set_monitor monitor=%p", monitor); + OBSData data = obs_data_create(); + obs_data_release(data); + + if (target_data.count() < 1) + return; + + obs_data_set_string(data, "source_name", target_data[0].toByteArray().constData()); + + obs_data_set_string(data, "filter_name", + target_data.count() > 1 ? target_data[1].toByteArray().constData() : ""); + + obs_source_update(monitor, data); + obs_source_update_properties(monitor); +} + void FTDock::updateState() { OBSSource target = get_source(); @@ -249,6 +266,16 @@ void FTDock::updateState() } } calldata_free(&cd); + + if (!data) + return; + + pthread_mutex_lock(&data->mutex); + + if (data->src_monitor) + set_monitor(data->src_monitor, targetSelector->currentData().toList()); + + pthread_mutex_unlock(&data->mutex); } void FTDock::pauseButtonChanged(int state) @@ -370,3 +397,11 @@ void FTDock::load_properties(obs_data_t *props) // TODO: implement me // pthread_mutex_unlock(&data->mutex); } + +void face_tracker_dock_destroy(struct face_tracker_dock_s *data) +{ + obs_display_destroy(data->disp); + data->disp = NULL; + obs_source_release(data->src_monitor); + bfree(data); +} diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index 95131fa..5f9898c 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -22,6 +22,7 @@ class FTDock : public QDockWidget { class QComboBox *targetSelector; class QCheckBox *pauseButton; class QPushButton *resetButton; + class FTWidget *ftWidget; public: FTDock(QWidget *parent = nullptr); @@ -32,8 +33,6 @@ class FTDock : public QDockWidget { void save_properties(obs_data_t*); void load_properties(obs_data_t*); - void SetWidget(class FTWidget *w); - private: void showEvent(QShowEvent *event) override; void hideEvent(QHideEvent *event) override; diff --git a/ui/face-tracker-widget.cpp b/ui/face-tracker-widget.cpp index 1c0b7ea..c8b08bf 100644 --- a/ui/face-tracker-widget.cpp +++ b/ui/face-tracker-widget.cpp @@ -18,25 +18,49 @@ static void draw(void *param, uint32_t cx, uint32_t cy) { auto *data = (struct face_tracker_dock_s *)param; - if (pthread_mutex_trylock(&data->mutex)) return; - //const auto src_shown = data->src_shown; - gs_blend_state_push(); gs_reset_blend_state(); + if (data->src_monitor) { + int w_src = obs_source_get_width(data->src_monitor); + int h_src = obs_source_get_height(data->src_monitor); + if (w_src <= 0 || h_src <= 0) + goto err; + int w, h; + if (w_src * cy > h_src * cx) { + w = cx; + h = cx * h_src / w_src; + } else { + h = cy; + w = cy * w_src / h_src; + } + + gs_projection_push(); + gs_viewport_push(); + + gs_set_viewport((cx-w)*0.5, (cy-h)*0.5, w, h); + gs_ortho(0.0f, w_src, -1.0f, h_src, -100.0f, 100.0f); + + obs_source_video_render(data->src_monitor); + + gs_viewport_pop(); + gs_projection_pop(); + } +err: + gs_blend_state_pop(); pthread_mutex_unlock(&data->mutex); } -FTWidget::FTWidget(QWidget *parent) +FTWidget::FTWidget(struct face_tracker_dock_s *data_, QWidget *parent) : QWidget(parent) , eventFilter(BuildEventFilter()) { - data = NULL; + face_tracker_dock_addref((data = data_)); setAttribute(Qt::WA_PaintOnScreen); setAttribute(Qt::WA_StaticContents); setAttribute(Qt::WA_NoSystemBackground); @@ -54,13 +78,6 @@ FTWidget::~FTWidget() face_tracker_dock_release(data); } -void FTWidget::SetData(struct face_tracker_dock_s *d) -{ - face_tracker_dock_addref(d); - face_tracker_dock_release(data); - data = d; -} - OBSEventFilter *FTWidget::BuildEventFilter() { return new OBSEventFilter([this](QObject *obj, QEvent *event) { @@ -93,6 +110,8 @@ OBSEventFilter *FTWidget::BuildEventFilter() void FTWidget::CreateDisplay() { + if (!data) + return; if (data->disp || !windowHandle()->isExposed()) return; diff --git a/ui/face-tracker-widget.hpp b/ui/face-tracker-widget.hpp index 8a76d50..e0a02da 100644 --- a/ui/face-tracker-widget.hpp +++ b/ui/face-tracker-widget.hpp @@ -27,9 +27,8 @@ class FTWidget : public QWidget { OBSEventFilter *BuildEventFilter(); public: - FTWidget(QWidget *parent); + FTWidget(struct face_tracker_dock_s *data, QWidget *parent); ~FTWidget(); - void SetData(struct face_tracker_dock_s *data); void setShown(bool shown); private: From 2b03a211d0536bbd253ab88ac7bf15dac3723904 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Tue, 28 Dec 2021 19:44:02 +0900 Subject: [PATCH 09/19] dock: Save last state of target selection --- ui/face-tracker-dock-internal.hpp | 8 +-- ui/face-tracker-dock.cpp | 107 ++++++++++++++++++++++++++---- ui/face-tracker-dock.hpp | 4 ++ 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/ui/face-tracker-dock-internal.hpp b/ui/face-tracker-dock-internal.hpp index daca7c9..a53f55b 100644 --- a/ui/face-tracker-dock-internal.hpp +++ b/ui/face-tracker-dock-internal.hpp @@ -12,13 +12,6 @@ struct face_tracker_dock_s obs_source_t *src_monitor; }; -static inline struct face_tracker_dock_s *face_tracker_dock_create() -{ - struct face_tracker_dock_s *data = (struct face_tracker_dock_s *)bzalloc(sizeof(struct face_tracker_dock_s)); - data->ref = 1; - return data; -} - static inline void face_tracker_dock_addref(struct face_tracker_dock_s *data) { if (!data) @@ -26,6 +19,7 @@ static inline void face_tracker_dock_addref(struct face_tracker_dock_s *data) os_atomic_inc_long(&data->ref); } +struct face_tracker_dock_s *face_tracker_dock_create(); void face_tracker_dock_destroy(struct face_tracker_dock_s *data); static inline void face_tracker_dock_release(struct face_tracker_dock_s *data) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index d02ddcb..3cea5ec 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -60,22 +60,42 @@ struct init_target_selector_s { QComboBox *q; int index; + const char *source_name; + const char *filter_name; }; -void init_target_selector_cb_add(struct init_target_selector_s *ctx, obs_source_t *source, obs_source_t *filter) +static bool init_target_selector_compare_name(struct init_target_selector_s *ctx, const char *source_name, const char *filter_name) +{ + if (strcmp(ctx->source_name, source_name) != 0) + return false; + + bool ctx_no_filter = !ctx->filter_name; + bool cur_no_filter = !filter_name; + + if (ctx_no_filter && cur_no_filter) + return true; + + if (ctx_no_filter != cur_no_filter) + return false; + + return strcmp(ctx->filter_name, filter_name) == 0; +} + +static void init_target_selector_cb_add(struct init_target_selector_s *ctx, obs_source_t *source, obs_source_t *filter) { QString text; QList val; const char *name = obs_source_get_name(source); + const char *filter_name = NULL; text = QString::fromUtf8(name); val.append(QVariant(name)); if (filter) { - const char *name = obs_source_get_name(filter); + filter_name = obs_source_get_name(filter); text += " / "; - text += QString::fromUtf8(name); - val.append(QVariant(name)); + text += QString::fromUtf8(filter_name); + val.append(QVariant(filter_name)); } if (ctx->index < ctx->q->count()) { @@ -84,6 +104,12 @@ void init_target_selector_cb_add(struct init_target_selector_s *ctx, obs_source_ } else ctx->q->insertItem(ctx->index, text, QVariant(val)); + + if (ctx->source_name) { + if (init_target_selector_compare_name(ctx, name, filter_name)) + ctx->q->setCurrentIndex(ctx->index); + } + ctx->index++; } @@ -112,18 +138,21 @@ static bool init_target_selector_cb_source(void *data, obs_source_t *source) return true; } -static void init_target_selector(QComboBox *q) +static void init_target_selector(QComboBox *q, const char *source_name=NULL, const char *filter_name=NULL) { QString current = q->currentText(); - init_target_selector_s ctx = {q, 0}; + if (filter_name && !*filter_name) + filter_name = NULL; + + init_target_selector_s ctx = {q, 0, source_name, filter_name}; obs_enum_scenes(init_target_selector_cb_source, &ctx); obs_enum_sources(init_target_selector_cb_source, &ctx); while (q->count() > ctx.index) q->removeItem(ctx.index); - if (current.length()) { + if (current.length() && !source_name) { int ix = q->findText(current); if (ix >= 0) q->setCurrentIndex(ix); @@ -178,6 +207,8 @@ FTDock::FTDock(QWidget *parent) connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); updateState(); + connect(this, &FTDock::dataChanged, this, &FTDock::updateWidget); + obs_frontend_add_event_callback(frontendEvent_cb, this); } @@ -212,6 +243,9 @@ void FTDock::frontendEvent(enum obs_frontend_event event) void FTDock::targetSelectorChanged() { + if (updating_widget) + return; + updateState(); } @@ -247,7 +281,6 @@ static inline void set_monitor(obs_source_t *monitor, const QList &tar target_data.count() > 1 ? target_data[1].toByteArray().constData() : ""); obs_source_update(monitor, data); - obs_source_update_properties(monitor); } void FTDock::updateState() @@ -278,6 +311,27 @@ void FTDock::updateState() pthread_mutex_unlock(&data->mutex); } +void FTDock::updateWidget() +{ + if (updating_widget) + return; + updating_widget = true; + pthread_mutex_lock(&data->mutex); + + if (data->src_monitor) { + obs_data_t *props = obs_source_get_settings(data->src_monitor); + const char *source_name = obs_data_get_string(props, "source_name"); + const char *filter_name = obs_data_get_string(props, "filter_name"); + + init_target_selector(targetSelector, source_name, filter_name); + + obs_data_release(props); + } + + pthread_mutex_unlock(&data->mutex); + updating_widget = false; +} + void FTDock::pauseButtonChanged(int state) { OBSSource target = get_source(); @@ -386,16 +440,40 @@ void FTDock::save_properties(obs_data_t *props) // Save indicates a source or a filter has been changed. scenesMayChanged(); - // pthread_mutex_lock(&data->mutex); - // TODO: implement me - // pthread_mutex_unlock(&data->mutex); + pthread_mutex_lock(&data->mutex); + + obs_data_t *prop = obs_source_get_settings(data->src_monitor); + if (prop) { + obs_data_set_obj(props, "monitor", prop); + obs_data_release(prop); + } + + pthread_mutex_unlock(&data->mutex); } void FTDock::load_properties(obs_data_t *props) { - // pthread_mutex_lock(&data->mutex); - // TODO: implement me - // pthread_mutex_unlock(&data->mutex); + pthread_mutex_lock(&data->mutex); + + if (data && data->src_monitor) { + obs_data_t *prop = obs_data_get_obj(props, "monitor"); + if (prop) { + obs_source_update(data->src_monitor, prop); + obs_data_release(prop); + } + } + + pthread_mutex_unlock(&data->mutex); + + dataChanged(); +} + +struct face_tracker_dock_s *face_tracker_dock_create() +{ + struct face_tracker_dock_s *data = (struct face_tracker_dock_s *)bzalloc(sizeof(struct face_tracker_dock_s)); + data->ref = 1; + pthread_mutex_init(&data->mutex, NULL); + return data; } void face_tracker_dock_destroy(struct face_tracker_dock_s *data) @@ -403,5 +481,6 @@ void face_tracker_dock_destroy(struct face_tracker_dock_s *data) obs_display_destroy(data->disp); data->disp = NULL; obs_source_release(data->src_monitor); + pthread_mutex_destroy(&data->mutex); bfree(data); } diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index 5f9898c..bac3f2b 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -24,6 +24,8 @@ class FTDock : public QDockWidget { class QPushButton *resetButton; class FTWidget *ftWidget; + bool updating_widget = false; + public: FTDock(QWidget *parent = nullptr); ~FTDock(); @@ -44,10 +46,12 @@ class FTDock : public QDockWidget { signals: void scenesMayChanged(); + void dataChanged(); public slots: void checkTargetSelector(); void updateState(); + void updateWidget(); private slots: void targetSelectorChanged(); From 002e93d3c5d1af125501954ceac22cdc1f74d1ee Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Tue, 28 Dec 2021 19:54:26 +0900 Subject: [PATCH 10/19] dock: Add notrack checkbox for the monitor --- ui/face-tracker-dock.cpp | 19 ++++++++++++++++++- ui/face-tracker-dock.hpp | 2 ++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index 3cea5ec..1e17960 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -202,6 +202,10 @@ FTDock::FTDock(QWidget *parent) ftWidget = new FTWidget(data, this); mainLayout->addWidget(ftWidget); + notrackButton = new QCheckBox(obs_module_text("Show all region"), this); + mainLayout->addWidget(notrackButton); + connect(notrackButton, &QCheckBox::stateChanged, this, &FTDock::notrackButtonChanged); + setWidget(dockWidgetContents); connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); @@ -320,11 +324,14 @@ void FTDock::updateWidget() if (data->src_monitor) { obs_data_t *props = obs_source_get_settings(data->src_monitor); + const char *source_name = obs_data_get_string(props, "source_name"); const char *filter_name = obs_data_get_string(props, "filter_name"); - init_target_selector(targetSelector, source_name, filter_name); + bool notrack = obs_data_get_bool(props, "notrack"); + notrackButton->setCheckState(notrack ? Qt::Checked : Qt::Unchecked); + obs_data_release(props); } @@ -360,6 +367,16 @@ void FTDock::resetButtonClicked(bool checked) calldata_free(&cd); } +void FTDock::notrackButtonChanged(int state) +{ + if (!data || !data->src_monitor) + return; + obs_data_t *props = obs_data_create(); + obs_data_set_bool(props, "notrack", state==Qt::Checked); + obs_source_update(data->src_monitor, props); + obs_data_release(props); +} + static void save_load_ft_docks(obs_data_t *save_data, bool saving, void *) { blog(LOG_INFO, "save_load_ft_docks saving=%d", (int)saving); diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index bac3f2b..1871bdf 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -23,6 +23,7 @@ class FTDock : public QDockWidget { class QCheckBox *pauseButton; class QPushButton *resetButton; class FTWidget *ftWidget; + class QCheckBox *notrackButton; bool updating_widget = false; @@ -57,6 +58,7 @@ private slots: void targetSelectorChanged(); void pauseButtonChanged(int state); void resetButtonClicked(bool checked); + void notrackButtonChanged(int state); }; extern "C" { From b79b440f1588194a0872e8151459df297e356bce Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Fri, 10 Dec 2021 12:17:18 +0900 Subject: [PATCH 11/19] Build devel branch --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d3c3e54..832dd0e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,6 +14,7 @@ trigger: include: - master - main + - devel tags: include: - '*' From db88fead9af03f13451e1cced91ac44d940c2356 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Wed, 29 Dec 2021 18:33:22 +0900 Subject: [PATCH 12/19] monitor: Suppress log messages When the target source or filter was not found, messages were displayed every tick of the video. This commit reduces at most 2 times. --- src/face-tracker-monitor.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/face-tracker-monitor.cpp b/src/face-tracker-monitor.cpp index 440cc70..c2a2390 100644 --- a/src/face-tracker-monitor.cpp +++ b/src/face-tracker-monitor.cpp @@ -3,6 +3,8 @@ #include "plugin-macros.generated.h" +#define MAX_ERROR 2 + struct face_tracker_monitor { obs_source_t *context; @@ -16,6 +18,8 @@ struct face_tracker_monitor obs_weak_source_t *source_ref; obs_weak_source_t *filter_ref; + + int n_error; }; static const char *ftmon_get_name(void *unused) @@ -55,14 +59,17 @@ static void ftmon_update(void *data, obs_data_t *settings) if (source_name && (!s->source_name || strcmp(source_name, s->source_name))) { bfree(s->source_name); s->source_name = bstrdup(source_name); + s->n_error = 0; } if (!filter_name || !*filter_name) { bfree(s->filter_name); s->filter_name = NULL; + s->n_error = 0; } else if (!s->filter_name || strcmp(filter_name, s->filter_name)) { bfree(s->filter_name); s->filter_name = bstrdup(filter_name); + s->n_error = 0; } s->notrack = obs_data_get_bool(settings, "notrack"); @@ -168,10 +175,19 @@ static void ftmon_tick(void *data, float second) tick_source(s, s->filter_ref, s->filter_name, get_filter_by_name); if (source_specified && !s->source_ref) { - blog(LOG_INFO, "failed to get source \"%s\"", s->source_name); + if (s->n_error < MAX_ERROR) { + blog(LOG_INFO, "failed to get source \"%s\"", s->source_name); + s->n_error ++; + } } else if (filter_specified && !s->filter_ref) { - blog(LOG_INFO, "failed to get filter \"%s\"", s->filter_name); + if (s->n_error < MAX_ERROR) { + blog(LOG_INFO, "failed to get filter \"%s\"", s->filter_name); + s->n_error ++; + } + } + else { + s->n_error = 0; } } From 1bd35b85ed0ae8cd611610a6aa99be8535457136 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Wed, 29 Dec 2021 21:01:36 +0900 Subject: [PATCH 13/19] dock: Add properties button The bahvior is differently implemented for sources and filters. For the source, it opens the properties dialog for that source. For the filter, since API does not provide to open the properties of the filter itself, the button will open the filters dialog for the parent source. --- ui/face-tracker-dock.cpp | 28 ++++++++++++++++++++++++++++ ui/face-tracker-dock.hpp | 11 +++++++++++ 2 files changed, 39 insertions(+) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index 1e17960..0bb5b48 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -199,6 +199,12 @@ FTDock::FTDock(QWidget *parent) mainLayout->addWidget(resetButton); connect(resetButton, &QPushButton::clicked, this, &FTDock::resetButtonClicked); +#ifdef HAVE_PROPERTY_BUTTON + propertyButton = new QPushButton(obs_module_text("Properties"), this); + mainLayout->addWidget(propertyButton); + connect(propertyButton, &QPushButton::clicked, this, &FTDock::propertyButtonClicked); +#endif + ftWidget = new FTWidget(data, this); mainLayout->addWidget(ftWidget); @@ -367,6 +373,28 @@ void FTDock::resetButtonClicked(bool checked) calldata_free(&cd); } +#ifdef HAVE_PROPERTY_BUTTON +void FTDock::propertyButtonClicked(bool checked) +{ + UNUSED_PARAMETER(checked); + + QList data = targetSelector->currentData().toList(); + if (data.count() < 1) + return; + + const char *name = data[0].toByteArray().constData(); + + obs_source_t *target = obs_get_source_by_name(name); + + if (data.count() == 1) + obs_frontend_open_source_properties(target); + else + obs_frontend_open_source_filters(target); + + obs_source_release(target); +} +#endif // HAVE_PROPERTY_BUTTON + void FTDock::notrackButtonChanged(int state) { if (!data || !data->src_monitor) diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp index 1871bdf..83fe56b 100644 --- a/ui/face-tracker-dock.hpp +++ b/ui/face-tracker-dock.hpp @@ -9,6 +9,11 @@ #include #include +// the necessary APIs are implemented at 27.0.1-107-g5b18faeb4. +#if LIBOBS_API_VER > MAKE_SEMANTIC_VERSION(27, 0, 1) +#define HAVE_PROPERTY_BUTTON +#endif + class FTDock : public QDockWidget { Q_OBJECT @@ -22,6 +27,9 @@ class FTDock : public QDockWidget { class QComboBox *targetSelector; class QCheckBox *pauseButton; class QPushButton *resetButton; +#ifdef HAVE_PROPERTY_BUTTON + class QPushButton *propertyButton; +#endif class FTWidget *ftWidget; class QCheckBox *notrackButton; @@ -58,6 +66,9 @@ private slots: void targetSelectorChanged(); void pauseButtonChanged(int state); void resetButtonClicked(bool checked); +#ifdef HAVE_PROPERTY_BUTTON + void propertyButtonClicked(bool checked); +#endif void notrackButtonChanged(int state); }; From c71749c866a37accbf9610a57e982a412adae99e Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Thu, 30 Dec 2021 19:13:03 +0900 Subject: [PATCH 14/19] fixup! dock: Save last state of target selection --- ui/face-tracker-dock.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp index 0bb5b48..8f9e0be 100644 --- a/ui/face-tracker-dock.cpp +++ b/ui/face-tracker-dock.cpp @@ -69,13 +69,12 @@ static bool init_target_selector_compare_name(struct init_target_selector_s *ctx if (strcmp(ctx->source_name, source_name) != 0) return false; - bool ctx_no_filter = !ctx->filter_name; - bool cur_no_filter = !filter_name; - - if (ctx_no_filter && cur_no_filter) + // both expect source + if (!ctx->filter_name && !filter_name) return true; - if (ctx_no_filter != cur_no_filter) + // either one expects source + if (!ctx->filter_name || !filter_name) return false; return strcmp(ctx->filter_name, filter_name) == 0; From eff5a6665884462acf8315c75a09310f7912daa5 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Fri, 31 Dec 2021 20:12:30 +0900 Subject: [PATCH 15/19] Add option to save data to file This function will save data file that is compatible with gnuplot. --- CMakeLists.txt | 1 + doc/properties-ptz.md | 61 +++++++++++++++++++++++++++++----------- doc/properties.md | 29 +++++++++++++++++++ src/face-tracker-ptz.cpp | 55 +++++++++++++++++++++++++++++++----- src/face-tracker-ptz.hpp | 6 ++++ src/face-tracker.cpp | 45 +++++++++++++++++++++++++---- src/face-tracker.hpp | 6 ++++ src/helper.cpp | 29 +++++++++++++++++++ src/helper.hpp | 8 ++++++ 9 files changed, 212 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ad4b7a..0b6d0c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,7 @@ set(LINUX_MAINTAINER_EMAIL "norihiro@nagater.net") option(WITH_PTZ_TCP "Enable to connect PTZ camera through TCP socket" ON) option(ENABLE_MONITOR_USER "Enable monitor source for user" OFF) option(WITH_DOCK "Enable dock" ON) +option(ENABLE_DEBUG_DATA "Enable property to save error and control data" OFF) set(CMAKE_PREFIX_PATH "${QTDIR}") set(CMAKE_AUTOMOC ON) diff --git a/doc/properties-ptz.md b/doc/properties-ptz.md index 3d17fb4..97e79c2 100644 --- a/doc/properties-ptz.md +++ b/doc/properties-ptz.md @@ -109,22 +109,6 @@ The nonlinear band makes smooth connection from the dead band to the linear rang ### Attenuation time for lost face After the face is lost, integral term will be attenuated by this time. The dimension is time and the unit is s. -## Debug -These properties enables how the face detection and tracking works. -Note that these features are automatically turned off when the source is displayed on the program of OBS Studio. -You can keep enable the checkboxes and keep monitoring the detection accuracy before the scene goes to the program. - -### Show face detection results -**Deprecated** -If enabled, face detection and tracking results are shown. -The face detection results are displayed in blue boxes. -The Tracking results are displayed in green boxes. - -### Always show information -**Deprecated** -If enabled, debugging properties listed above are effective even if the source is displayed on the program. -This will be useful to make a demonstration of face-tracker itself. - ## Output This property group configure how to connect to the PTZ camera. @@ -155,3 +139,48 @@ It might be useful if you camera is mounted on ceil. Just in c ase the zoom control behave opposite directon, check this. You should not check this in most cases. This is a deplicated option. + +## Debug +These properties enables how the face detection and tracking works. +Note that these features are automatically turned off when the source is displayed on the program of OBS Studio. +You can keep enable the checkboxes and keep monitoring the detection accuracy before the scene goes to the program. + +### Show face detection results +**Deprecated** +If enabled, face detection and tracking results are shown. +The face detection results are displayed in blue boxes. +The Tracking results are displayed in green boxes. + +### Always show information +**Deprecated** +If enabled, debugging properties listed above are effective even if the source is displayed on the program. +This will be useful to make a demonstration of face-tracker itself. + +### Save correlation tracker, calculated error, control data to file +**Not available for released version** +Save internal calculation into the specified file for each. +This option is not available without building with `ENABLE_DEBUG_DATA` +but still can be set through obs-websocket or manually editing the scene file to add a text property with a file name to be written. +To disable it back, remove the property or set zero-length text. + +#### Correlation tracker +Property name: `debug_data_tracker` + +The data contains time in second, 3 coordinates (X, Y, Z), and score of the correlation tracker. +The X and Y coordinates are the center of the face. +The Z coordinate is a square-root of the area. +Sometimes multiple correlation trackers run at the same time. In that case, multiple lines are written at the same timing. + +#### Calculated error +Property name: `debug_data_error` + +The data contains time in second, 3 coordinates (X, Y, Z). +The calculated error is the adjusted measure with current resolution, the cropped region when the frame was rendered, and user-specified tracking target. +0-value indicates the face is well aligned and positive or negative value indicates the cropped region need to be moved. + +#### Control +Property name: `debug_data_control` + +The data contains time in second, 3 coordinates (X, Y, Z), and another set of 3 coordinates. +The first set of the coordinates is a linear floating-point value of the control signal. +The second set of the coordinates is an integer value that should go to the PTZ device. diff --git a/doc/properties.md b/doc/properties.md index 7f5a443..5c13344 100644 --- a/doc/properties.md +++ b/doc/properties.md @@ -147,3 +147,32 @@ This is useful to check how much margins are there around the cropped area. ### Always show information If enabled, debugging properties listed above are effective even if the source is displayed on the program. This will be useful to make a demonstration of face-tracker itself. + +### Save correlation tracker, calculated error, control data to file +**Not available for released version** +Save internal calculation into the specified file for each. +This option is not available without building with `ENABLE_DEBUG_DATA` +but still can be set through obs-websocket or manually editing the scene file to add a text property with a file name to be written. +To disable it back, remove the property or set zero-length text. + +#### Correlation tracker +Property name: `debug_data_tracker` + +The data contains time in second, 3 coordinates (X, Y, Z), and score of the correlation tracker. +The X and Y coordinates are the center of the face. +The Z coordinate is a square-root of the area. +Sometimes multiple correlation trackers run at the same time. In that case, multiple lines are written at the same timing. + +#### Calculated error +Property name: `debug_data_error` + +The data contains time in second, 3 coordinates (X, Y, Z). +The calculated error is the adjusted measure with current resolution, the cropped region when the frame was rendered, and user-specified tracking target. +0-value indicates the face is well aligned and positive or negative value indicates the cropped region need to be moved. + +#### Control +Property name: `debug_data_control` + +The data contains time in second, 3 coordinates (X, Y, Z). +The X and Y coordinates are the center of the cropped region. +The Z coordinate is a square-root of the area of the cropped region. diff --git a/src/face-tracker-ptz.cpp b/src/face-tracker-ptz.cpp index 09d5ddb..1e8b71f 100644 --- a/src/face-tracker-ptz.cpp +++ b/src/face-tracker-ptz.cpp @@ -18,8 +18,6 @@ #endif #include "dummy-backend.hpp" -#define debug_track(fmt, ...) // blog(LOG_INFO, fmt, __VA_ARGS__) - #define PTZ_MAX_X 0x18 #define PTZ_MAX_Y 0x14 #define PTZ_MAX_Z 0x07 @@ -210,6 +208,10 @@ static void ftptz_update(void *data, obs_data_t *settings) s->debug_notrack = obs_data_get_bool(settings, "debug_notrack"); s->debug_always_show = obs_data_get_bool(settings, "debug_always_show"); + debug_data_open(&s->debug_data_tracker, &s->debug_data_tracker_last, settings, "debug_data_tracker"); + debug_data_open(&s->debug_data_error, &s->debug_data_error_last, settings, "debug_data_error"); + debug_data_open(&s->debug_data_control, &s->debug_data_control_last, settings, "debug_data_control"); + s->ptz_max_x = obs_data_get_int(settings, "ptz_max_x"); s->ptz_max_y = obs_data_get_int(settings, "ptz_max_y"); s->ptz_max_z = obs_data_get_int(settings, "ptz_max_z"); @@ -280,6 +282,15 @@ static void ftptz_destroy(void *data) delete s->ftm; bfree(s->ptz_type); + if (s->debug_data_tracker) + fclose(s->debug_data_tracker); + if (s->debug_data_error) + fclose(s->debug_data_error); + if (s->debug_data_control) + fclose(s->debug_data_control); + bfree(s->debug_data_tracker_last); + bfree(s->debug_data_error_last); + bfree(s->debug_data_control_last); bfree(s); } @@ -396,8 +407,6 @@ static obs_properties_t *ftptz_properties(void *data) { obs_properties_t *pp = obs_properties_create(); - obs_properties_add_bool(pp, "debug_faces", "Show face detection results"); - obs_properties_add_bool(pp, "debug_always_show", "Always show information (useful for demo)"); obs_property_t *p = obs_properties_add_list(pp, "ptz-type", obs_module_text("PTZ Type"), OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING); obs_property_list_add_string(p, obs_module_text("None"), "dummy"); obs_property_list_add_string(p, obs_module_text("through PTZ Controls"), "obsptz"); @@ -417,6 +426,21 @@ static obs_properties_t *ftptz_properties(void *data) obs_properties_add_group(props, "output", obs_module_text("Output"), OBS_GROUP_NORMAL, pp); } + { + obs_properties_t *pp = obs_properties_create(); + obs_properties_add_bool(pp, "debug_faces", "Show face detection results"); + obs_properties_add_bool(pp, "debug_always_show", "Always show information (useful for demo)"); +#ifdef ENABLE_DEBUG_DATA + obs_properties_add_path(pp, "debug_data_tracker", "Save correlation tracker data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_error", "Save calculated error data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_control", "Save control data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); +#endif + obs_properties_add_group(props, "debug", obs_module_text("Debugging"), OBS_GROUP_NORMAL, pp); + } + return props; } @@ -643,7 +667,13 @@ static void tick_filter(struct face_tracker_ptz *s, float second) else if (n > +u_max[i]) n = +u_max[i]; s->u[i] = n; } - debug_track("tick_filter: u: %d %d %d uf: %f %f %f", s->u[0], s->u[1], s->u[2], uf.v[0], uf.v[1], uf.v[2]); + + if (s->debug_data_control) { + fprintf(s->debug_data_control, "%f\t%f\t%f\t%f\t%d\t%d\t%d\n", + os_gettime_ns() * 1e-9, + uf.v[0], uf.v[1], uf.v[2], + s->u[0], s->u[1], s->u[2] ); + } } static void ftf_activate(void *data) @@ -734,6 +764,7 @@ static inline void calculate_error(struct face_tracker_ptz *s) auto &tracker_rects = s->ftm->tracker_rects; for (int i=0; iftm->landmark_detection_data) { pointf_s center = landmark_center(tracker_rects[i].landmark); @@ -746,14 +777,18 @@ static inline void calculate_error(struct face_tracker_ptz *s) r.v[2] = sqrtf(area * (float)(4.0f / M_PI)); } + if (s->debug_data_tracker) { + fprintf(s->debug_data_tracker, "%f\t%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + r.v[0], r.v[1], r.v[2], score ); + } + r.v[0] -= get_width(tracker_rects[i].crop_rect) * s->track_x; r.v[1] += get_height(tracker_rects[i].crop_rect) * s->track_y; r.v[2] /= s->track_z; f3 w (tracker_rects[i].crop_rect); - float score = tracker_rects[i].rect.score; f3 e = (r-w) * score; - debug_track("calculate_error: %d %f %f %f %f", i, e.v[0], e.v[1], e.v[2], score); if (score>0.0f && !isnan(e)) { e_tot += e; sc_tot += score; @@ -766,6 +801,12 @@ static inline void calculate_error(struct face_tracker_ptz *s) else s->detect_err = f3(0, 0, 0); s->face_found = found; + + if (s->debug_data_error) { + fprintf(s->debug_data_error, "%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + s->detect_err.v[0], s->detect_err.v[1], s->detect_err.v[2] ); + } } static struct obs_source_frame *ftptz_filter_video(void *data, struct obs_source_frame *frame) diff --git a/src/face-tracker-ptz.hpp b/src/face-tracker-ptz.hpp index 35eb7c7..bc65420 100644 --- a/src/face-tracker-ptz.hpp +++ b/src/face-tracker-ptz.hpp @@ -33,6 +33,12 @@ struct face_tracker_ptz bool debug_faces; bool debug_notrack; bool debug_always_show; + FILE *debug_data_tracker; + FILE *debug_data_error; + FILE *debug_data_control; + char *debug_data_tracker_last; + char *debug_data_error_last; + char *debug_data_control_last; char *ptz_type; int ptz_max_x, ptz_max_y, ptz_max_z; diff --git a/src/face-tracker.cpp b/src/face-tracker.cpp index d9a07c5..f26248b 100644 --- a/src/face-tracker.cpp +++ b/src/face-tracker.cpp @@ -13,9 +13,6 @@ #include "face-tracker-manager.hpp" #include "source_list.h" -// #define debug_track(fmt, ...) blog(LOG_INFO, fmt, __VA_ARGS__) -#define debug_track(fmt, ...) - static gs_effect_t *effect_ft = NULL; static inline void scale_texture(struct face_tracker_filter *s, float scale); @@ -99,6 +96,10 @@ static void ftf_update(void *data, obs_data_t *settings) s->debug_faces = obs_data_get_bool(settings, "debug_faces"); s->debug_notrack = obs_data_get_bool(settings, "debug_notrack"); s->debug_always_show = obs_data_get_bool(settings, "debug_always_show"); + + debug_data_open(&s->debug_data_tracker, &s->debug_data_tracker_last, settings, "debug_data_tracker"); + debug_data_open(&s->debug_data_error, &s->debug_data_error_last, settings, "debug_data_error"); + debug_data_open(&s->debug_data_control, &s->debug_data_control_last, settings, "debug_data_control"); } static void fts_update(void *data, obs_data_t *settings) @@ -184,6 +185,15 @@ static void ftf_destroy(void *data) bfree(s->target_name); obs_weak_source_release(s->target_ref); + if (s->debug_data_tracker) + fclose(s->debug_data_tracker); + if (s->debug_data_error) + fclose(s->debug_data_error); + if (s->debug_data_control) + fclose(s->debug_data_control); + bfree(s->debug_data_tracker_last); + bfree(s->debug_data_error_last); + bfree(s->debug_data_control_last); bfree(s); } @@ -282,6 +292,14 @@ static obs_properties_t *ftf_properties(void *data) obs_properties_add_bool(pp, "debug_faces", "Show face detection results"); obs_properties_add_bool(pp, "debug_notrack", "Stop tracking faces"); obs_properties_add_bool(pp, "debug_always_show", "Always show information (useful for demo)"); +#ifdef ENABLE_DEBUG_DATA + obs_properties_add_path(pp, "debug_data_tracker", "Save correlation tracker data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_error", "Save calculated error data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_control", "Save control data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); +#endif obs_properties_add_group(props, "debug", obs_module_text("Debugging"), OBS_GROUP_NORMAL, pp); } @@ -380,6 +398,12 @@ static void tick_filter(struct face_tracker_filter *s, float second) } } + if (s->debug_data_control) { + fprintf(s->debug_data_control, "%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + u.v[0], u.v[1], u.v[2] ); + } + s->ftm->crop_cur = f3_to_rectf(u, s->width_with_aspect, s->height_with_aspect); } @@ -614,6 +638,7 @@ static inline void calculate_error(struct face_tracker_filter *s) auto &tracker_rects = s->ftm->tracker_rects; for (int i=0; iftm->landmark_detection_data) { pointf_s center = landmark_center(tracker_rects[i].landmark); @@ -626,15 +651,19 @@ static inline void calculate_error(struct face_tracker_filter *s) r.v[2] = sqrtf(area * (float)(4.0f / M_PI)); } + if (s->debug_data_tracker) { + fprintf(s->debug_data_tracker, "%f\t%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + r.v[0], r.v[1], r.v[2], score ); + } + r.v[0] -= get_width(tracker_rects[i].crop_rect) * s->track_x; r.v[1] += get_height(tracker_rects[i].crop_rect) * s->track_y; r.v[2] /= s->track_z; r = ensure_range(r, s); f3 w (tracker_rects[i].crop_rect); - float score = tracker_rects[i].rect.score; f3 e = (r-w) * score; - debug_track("calculate_error: %d %f %f %f %f", i, e.v[0], e.v[1], e.v[2], score); if (score>0.0f && !isnan(e)) { e_tot += e; sc_tot += score; @@ -646,6 +675,12 @@ static inline void calculate_error(struct face_tracker_filter *s) s->detect_err = e_tot * (1.0f / sc_tot); else s->detect_err = f3(0, 0, 0); + + if (s->debug_data_error) { + fprintf(s->debug_data_error, "%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + s->detect_err.v[0], s->detect_err.v[1], s->detect_err.v[2] ); + } } static inline void draw_sprite_crop(float width, float height, float x0, float y0, float x1, float y1); diff --git a/src/face-tracker.hpp b/src/face-tracker.hpp index 379da3d..fdfb21b 100644 --- a/src/face-tracker.hpp +++ b/src/face-tracker.hpp @@ -43,6 +43,12 @@ struct face_tracker_filter bool debug_faces; bool debug_notrack; bool debug_always_show; + FILE *debug_data_tracker; + FILE *debug_data_error; + FILE *debug_data_control; + char *debug_data_tracker_last; + char *debug_data_error_last; + char *debug_data_control_last; bool is_paused; obs_hotkey_pair_id hotkey_pause; diff --git a/src/helper.cpp b/src/helper.cpp index 60200a0..16e35d7 100644 --- a/src/helper.cpp +++ b/src/helper.cpp @@ -135,3 +135,32 @@ void draw_landmark(const std::vector &landmark) gs_render_stop(GS_LINES); } + +void debug_data_open(FILE **dest, char **last_name, obs_data_t *settings, const char *name) +{ + const char *debug_data = obs_data_get_string(settings, name); + + // If the file name is not changed, just return. + if (*last_name && debug_data && strcmp(*last_name, debug_data) == 0) + return; + + // If both file names are empty, just return. + if (!*last_name && (!debug_data || !*debug_data)) + return; + + if (*dest) + fclose(*dest); + *dest = NULL; + + if (*last_name) + bfree(*last_name); + *last_name = NULL; + + if (debug_data && *debug_data) { + *dest = fopen(debug_data, "a"); + if (!*dest) { + blog(LOG_ERROR, "%s: Failed to open file \"%s\"", name, debug_data); + } + *last_name = bstrdup(debug_data); + } +} diff --git a/src/helper.hpp b/src/helper.hpp index 0838c39..e419e3c 100644 --- a/src/helper.hpp +++ b/src/helper.hpp @@ -2,6 +2,12 @@ #include #include +#ifdef _WIN32 +#define DEBUG_DATA_PATH_FILTER "TSV Files (*.tsv);;Data Files (*.dat);;All Files (*.*)" +#else +#define DEBUG_DATA_PATH_FILTER "Data Files (*.dat);;TSV Files (*.tsv);;All Files (*.*)" +#endif + struct pointf_s { float x; @@ -98,3 +104,5 @@ inline double from_dB(double x) { return exp(x * (M_LN10/20)); } + +void debug_data_open(FILE **dest, char **last_name, obs_data_t *settings, const char *name); From f2be0b90e3892ddadc0ecbc31e7bcc6935af791e Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 15 Jan 2022 11:39:51 +0900 Subject: [PATCH 16/19] Add try-catch blocks for dlib threads A crash was reported on #99. The try-catch blocks are added with a hope it might catch something. --- src/face-detector-base.cpp | 10 +++++++++- src/face-tracker-base.cpp | 13 +++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/face-detector-base.cpp b/src/face-detector-base.cpp index 0326537..7d2ecc0 100644 --- a/src/face-detector-base.cpp +++ b/src/face-detector-base.cpp @@ -40,7 +40,15 @@ void *face_detector_base::thread_routine(void *p) base->lock(); while(!base->request_stop) { - base->detect_main(); + try { + base->detect_main(); + } + catch (std::exception &e) { + blog(LOG_ERROR, "detect_main: exception %s", e.what()); + } + catch (...) { + blog(LOG_ERROR, "detect_main: unknown exception"); + } pthread_cond_wait(&base->cond, &base->mutex); } base->unlock(); diff --git a/src/face-tracker-base.cpp b/src/face-tracker-base.cpp index 59633de..9306a04 100644 --- a/src/face-tracker-base.cpp +++ b/src/face-tracker-base.cpp @@ -39,8 +39,17 @@ void *face_tracker_base::thread_routine(void *p) base->lock(); while(!base->stop_requested) { - if (!base->suspend_requested) - base->track_main(); + if (!base->suspend_requested) { + try { + base->track_main(); + } + catch (std::exception &e) { + blog(LOG_ERROR, "track_main: exception %s", e.what()); + } + catch (...) { + blog(LOG_ERROR, "track_main: unknown exception"); + } + } pthread_cond_wait(&base->cond, &base->mutex); } base->stopped = 1; From d6026aa67371401418e6a7fc9c16d291159c46cc Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 15 Jan 2022 11:57:34 +0900 Subject: [PATCH 17/19] face-tracker-dlib: Check image size before correlation tracking If image size differs, new detection should be started. --- src/face-tracker-dlib.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/face-tracker-dlib.cpp b/src/face-tracker-dlib.cpp index 9c10c22..f74b778 100644 --- a/src/face-tracker-dlib.cpp +++ b/src/face-tracker-dlib.cpp @@ -14,6 +14,7 @@ struct face_tracker_dlib_private_s texture_object *tex; rect_s rect; dlib::correlation_tracker *tracker; + int tracker_nc, tracker_nr; dlib::shape_predictor sp; dlib::full_object_detection shape; float last_scale; @@ -104,8 +105,11 @@ void face_tracker_dlib::track_main() if (!p->tracker) p->tracker = new dlib::correlation_tracker(); + auto &img = p->tex->get_dlib_img(); dlib::rectangle r (p->rect.x0, p->rect.y0, p->rect.x1, p->rect.y1); - p->tracker->start_track(p->tex->get_dlib_img(), r); + p->tracker->start_track(img, r); + p->tracker_nc = img.nc(); + p->tracker_nr = img.nr(); p->score0 = p->rect.score; p->need_restart = false; p->pslr_max = 0.0f; @@ -117,7 +121,17 @@ void face_tracker_dlib::track_main() p->rect.score = 0.0f; } else { - float s = p->tracker->update(p->tex->get_dlib_img()); + auto &img = p->tex->get_dlib_img(); + if (img.nc() != p->tracker_nc || img.nr() != p->tracker_nr) { + blog(LOG_ERROR, "face_tracker_dlib::track_main: cannot run correlation-tracker with different image size %dx%d, expected %dx%d", + img.nc(), img.nr(), + p->tracker_nc, p->tracker_nr ); + p->rect.score = 0; + p->n_track += 1; // to return score=0 + return; + } + + float s = p->tracker->update(img); if (s>p->pslr_max) p->pslr_max = s; if (spslr_min) p->pslr_min = s; dlib::rectangle r = p->tracker->get_position(); From 7765a721ca5c879d4d54c96a6a2e9045ef33d4b1 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 15 Jan 2022 12:59:51 +0900 Subject: [PATCH 18/19] [WIP] Align texture size I don't know why but it fails if width is not a multiple of 4. --- src/face-tracker.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/face-tracker.cpp b/src/face-tracker.cpp index f26248b..7db69a0 100644 --- a/src/face-tracker.cpp +++ b/src/face-tracker.cpp @@ -690,10 +690,15 @@ static inline void scale_texture(struct face_tracker_filter *s, float scale) if (!s->texrender_scaled) s->texrender_scaled = gs_texrender_create(GS_R8, GS_ZS_NONE); const uint32_t cx = s->known_width / scale, cy = s->known_height / scale; + + const int align = 8; + uint32_t width = (cx + align - 1) / align * align; + uint32_t height = (cy + align - 1) / align * align; + gs_texrender_reset(s->texrender_scaled); gs_blend_state_push(); gs_blend_function(GS_BLEND_ONE, GS_BLEND_ZERO); - if (gs_texrender_begin(s->texrender_scaled, cx, cy)) { + if (gs_texrender_begin(s->texrender_scaled, width, height)) { gs_ortho(0.0f, (float)cx, 0.0f, (float)cy, -100.0f, 100.0f); gs_texture_t *tex = gs_texrender_get_texture(s->texrender); if (tex && effect_ft) { @@ -714,6 +719,10 @@ static inline int stage_to_surface(struct face_tracker_filter *s, float scale) if (width<=0 || height<=0) return 1; + const int align = 8; + width = (width + align - 1) / align * align; + height = (height + align - 1) / align * align; + gs_texture_t *tex = gs_texrender_get_texture(s->texrender_scaled); if (!tex) return 2; From ef604bebcca66020c51f15fb811c28a812490b25 Mon Sep 17 00:00:00 2001 From: Norihiro Kamae Date: Sat, 15 Jan 2022 13:33:11 +0900 Subject: [PATCH 19/19] ci(windows): Fix failure of patch to slim dlib --- ci/windows/prepare-windows.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/windows/prepare-windows.cmd b/ci/windows/prepare-windows.cmd index e8adebb..bc51296 100644 --- a/ci/windows/prepare-windows.cmd +++ b/ci/windows/prepare-windows.cmd @@ -6,7 +6,7 @@ if not exist dlib ( git describe --tags --abbrev=0 --exclude="*-rc*" > dlib-tag.txt set /p dlibTag=<"dlib-tag.txt" git checkout %dlibTag% - patch -p1 -i ..\ci\common\dlib-slim.patch + git apply ../ci/common/dlib-slim.patch cd .. )