diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c5cc39..0b6d0c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,8 @@ set(LINUX_MAINTAINER_EMAIL "norihiro@nagater.net") option(WITH_PTZ_TCP "Enable to connect PTZ camera through TCP socket" ON) option(ENABLE_MONITOR_USER "Enable monitor source for user" OFF) +option(WITH_DOCK "Enable dock" ON) +option(ENABLE_DEBUG_DATA "Enable property to save error and control data" OFF) set(CMAKE_PREFIX_PATH "${QTDIR}") set(CMAKE_AUTOMOC ON) @@ -21,7 +23,7 @@ include(external/FindLibObs.cmake) endif() find_package(LibObs REQUIRED) -find_package(Qt5 REQUIRED COMPONENTS Core Widgets) +find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui) set(plugin_additional_libs) if (WITH_PTZ_TCP) @@ -101,6 +103,22 @@ if (WITH_PTZ_TCP) set(PLUGIN_HEADERS ${PLUGIN_HEADERS} src/libvisca-thread.hpp) endif() +if (WITH_DOCK) + set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + ui/face-tracker-dock.cpp + ui/face-tracker-widget.cpp + ) + set(PLUGIN_HEADERS + ${PLUGIN_HEADERS} + ui/face-tracker-dock.hpp + ui/face-tracker-widget.hpp + ) + set(plugin_additional_incs ${plugin_additional_incs} src) + set(plugin_additional_libs ${plugin_additional_libs} Qt5::Gui Qt5::GuiPrivate) + set(plugin_additional_incs ${plugin_additional_incs} ${Qt5Gui_PRIVATE_INCLUDE_DIRS}) +endif() + # --- Platform-independent build settings --- add_library(${CMAKE_PROJECT_NAME} MODULE ${PLUGIN_SOURCES} ${PLUGIN_HEADERS}) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d3c3e54..832dd0e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,6 +14,7 @@ trigger: include: - master - main + - devel tags: include: - '*' diff --git a/ci/linux/install-dependencies-ubuntu.sh b/ci/linux/install-dependencies-ubuntu.sh index fd5a987..0588c26 100755 --- a/ci/linux/install-dependencies-ubuntu.sh +++ b/ci/linux/install-dependencies-ubuntu.sh @@ -11,7 +11,8 @@ sudo apt-get install -y \ checkinstall \ cmake \ obs-studio \ - qtbase5-dev + qtbase5-dev \ + qtbase5-private-dev # Dirty hack obs_version="$(dpkg -s obs-studio | awk '$1=="Version:" {print gensub(/-.*/, "", 1, $2)}')" diff --git a/ci/windows/prepare-windows.cmd b/ci/windows/prepare-windows.cmd index e8adebb..bc51296 100644 --- a/ci/windows/prepare-windows.cmd +++ b/ci/windows/prepare-windows.cmd @@ -6,7 +6,7 @@ if not exist dlib ( git describe --tags --abbrev=0 --exclude="*-rc*" > dlib-tag.txt set /p dlibTag=<"dlib-tag.txt" git checkout %dlibTag% - patch -p1 -i ..\ci\common\dlib-slim.patch + git apply ../ci/common/dlib-slim.patch cd .. ) diff --git a/doc/properties-ptz.md b/doc/properties-ptz.md index 6719633..97e79c2 100644 --- a/doc/properties-ptz.md +++ b/doc/properties-ptz.md @@ -5,6 +5,8 @@ When clicked, tracking state is reset to the initial condition; reset internal s (This is not a property.) ## Preset +**Deprecated** +Preset will be provided through a dock. The preset group in the property dialog will be removed in the future release. ### Preset This combo-box sets the name of the preset to be loaded or saved. @@ -107,22 +109,6 @@ The nonlinear band makes smooth connection from the dead band to the linear rang ### Attenuation time for lost face After the face is lost, integral term will be attenuated by this time. The dimension is time and the unit is s. -## Debug -These properties enables how the face detection and tracking works. -Note that these features are automatically turned off when the source is displayed on the program of OBS Studio. -You can keep enable the checkboxes and keep monitoring the detection accuracy before the scene goes to the program. - -### Show face detection results -**Deprecated** -If enabled, face detection and tracking results are shown. -The face detection results are displayed in blue boxes. -The Tracking results are displayed in green boxes. - -### Always show information -**Deprecated** -If enabled, debugging properties listed above are effective even if the source is displayed on the program. -This will be useful to make a demonstration of face-tracker itself. - ## Output This property group configure how to connect to the PTZ camera. @@ -153,3 +139,48 @@ It might be useful if you camera is mounted on ceil. Just in c ase the zoom control behave opposite directon, check this. You should not check this in most cases. This is a deplicated option. + +## Debug +These properties enables how the face detection and tracking works. +Note that these features are automatically turned off when the source is displayed on the program of OBS Studio. +You can keep enable the checkboxes and keep monitoring the detection accuracy before the scene goes to the program. + +### Show face detection results +**Deprecated** +If enabled, face detection and tracking results are shown. +The face detection results are displayed in blue boxes. +The Tracking results are displayed in green boxes. + +### Always show information +**Deprecated** +If enabled, debugging properties listed above are effective even if the source is displayed on the program. +This will be useful to make a demonstration of face-tracker itself. + +### Save correlation tracker, calculated error, control data to file +**Not available for released version** +Save internal calculation into the specified file for each. +This option is not available without building with `ENABLE_DEBUG_DATA` +but still can be set through obs-websocket or manually editing the scene file to add a text property with a file name to be written. +To disable it back, remove the property or set zero-length text. + +#### Correlation tracker +Property name: `debug_data_tracker` + +The data contains time in second, 3 coordinates (X, Y, Z), and score of the correlation tracker. +The X and Y coordinates are the center of the face. +The Z coordinate is a square-root of the area. +Sometimes multiple correlation trackers run at the same time. In that case, multiple lines are written at the same timing. + +#### Calculated error +Property name: `debug_data_error` + +The data contains time in second, 3 coordinates (X, Y, Z). +The calculated error is the adjusted measure with current resolution, the cropped region when the frame was rendered, and user-specified tracking target. +0-value indicates the face is well aligned and positive or negative value indicates the cropped region need to be moved. + +#### Control +Property name: `debug_data_control` + +The data contains time in second, 3 coordinates (X, Y, Z), and another set of 3 coordinates. +The first set of the coordinates is a linear floating-point value of the control signal. +The second set of the coordinates is an integer value that should go to the PTZ device. diff --git a/doc/properties.md b/doc/properties.md index 0944b07..5c13344 100644 --- a/doc/properties.md +++ b/doc/properties.md @@ -5,6 +5,8 @@ When clicked, tracking state is reset to the initial condition; zero crop, reset (This is not a property.) ## Preset +**Deprecated** +Preset will be provided through a dock. The preset group in the property dialog will be removed in the future release. ### Preset This combo-box sets the name of the preset to be loaded or saved. @@ -145,3 +147,32 @@ This is useful to check how much margins are there around the cropped area. ### Always show information If enabled, debugging properties listed above are effective even if the source is displayed on the program. This will be useful to make a demonstration of face-tracker itself. + +### Save correlation tracker, calculated error, control data to file +**Not available for released version** +Save internal calculation into the specified file for each. +This option is not available without building with `ENABLE_DEBUG_DATA` +but still can be set through obs-websocket or manually editing the scene file to add a text property with a file name to be written. +To disable it back, remove the property or set zero-length text. + +#### Correlation tracker +Property name: `debug_data_tracker` + +The data contains time in second, 3 coordinates (X, Y, Z), and score of the correlation tracker. +The X and Y coordinates are the center of the face. +The Z coordinate is a square-root of the area. +Sometimes multiple correlation trackers run at the same time. In that case, multiple lines are written at the same timing. + +#### Calculated error +Property name: `debug_data_error` + +The data contains time in second, 3 coordinates (X, Y, Z). +The calculated error is the adjusted measure with current resolution, the cropped region when the frame was rendered, and user-specified tracking target. +0-value indicates the face is well aligned and positive or negative value indicates the cropped region need to be moved. + +#### Control +Property name: `debug_data_control` + +The data contains time in second, 3 coordinates (X, Y, Z). +The X and Y coordinates are the center of the cropped region. +The Z coordinate is a square-root of the area of the cropped region. diff --git a/src/face-detector-base.cpp b/src/face-detector-base.cpp index 0326537..7d2ecc0 100644 --- a/src/face-detector-base.cpp +++ b/src/face-detector-base.cpp @@ -40,7 +40,15 @@ void *face_detector_base::thread_routine(void *p) base->lock(); while(!base->request_stop) { - base->detect_main(); + try { + base->detect_main(); + } + catch (std::exception &e) { + blog(LOG_ERROR, "detect_main: exception %s", e.what()); + } + catch (...) { + blog(LOG_ERROR, "detect_main: unknown exception"); + } pthread_cond_wait(&base->cond, &base->mutex); } base->unlock(); diff --git a/src/face-tracker-base.cpp b/src/face-tracker-base.cpp index 59633de..9306a04 100644 --- a/src/face-tracker-base.cpp +++ b/src/face-tracker-base.cpp @@ -39,8 +39,17 @@ void *face_tracker_base::thread_routine(void *p) base->lock(); while(!base->stop_requested) { - if (!base->suspend_requested) - base->track_main(); + if (!base->suspend_requested) { + try { + base->track_main(); + } + catch (std::exception &e) { + blog(LOG_ERROR, "track_main: exception %s", e.what()); + } + catch (...) { + blog(LOG_ERROR, "track_main: unknown exception"); + } + } pthread_cond_wait(&base->cond, &base->mutex); } base->stopped = 1; diff --git a/src/face-tracker-dlib.cpp b/src/face-tracker-dlib.cpp index 9c10c22..f74b778 100644 --- a/src/face-tracker-dlib.cpp +++ b/src/face-tracker-dlib.cpp @@ -14,6 +14,7 @@ struct face_tracker_dlib_private_s texture_object *tex; rect_s rect; dlib::correlation_tracker *tracker; + int tracker_nc, tracker_nr; dlib::shape_predictor sp; dlib::full_object_detection shape; float last_scale; @@ -104,8 +105,11 @@ void face_tracker_dlib::track_main() if (!p->tracker) p->tracker = new dlib::correlation_tracker(); + auto &img = p->tex->get_dlib_img(); dlib::rectangle r (p->rect.x0, p->rect.y0, p->rect.x1, p->rect.y1); - p->tracker->start_track(p->tex->get_dlib_img(), r); + p->tracker->start_track(img, r); + p->tracker_nc = img.nc(); + p->tracker_nr = img.nr(); p->score0 = p->rect.score; p->need_restart = false; p->pslr_max = 0.0f; @@ -117,7 +121,17 @@ void face_tracker_dlib::track_main() p->rect.score = 0.0f; } else { - float s = p->tracker->update(p->tex->get_dlib_img()); + auto &img = p->tex->get_dlib_img(); + if (img.nc() != p->tracker_nc || img.nr() != p->tracker_nr) { + blog(LOG_ERROR, "face_tracker_dlib::track_main: cannot run correlation-tracker with different image size %dx%d, expected %dx%d", + img.nc(), img.nr(), + p->tracker_nc, p->tracker_nr ); + p->rect.score = 0; + p->n_track += 1; // to return score=0 + return; + } + + float s = p->tracker->update(img); if (s>p->pslr_max) p->pslr_max = s; if (spslr_min) p->pslr_min = s; dlib::rectangle r = p->tracker->get_position(); diff --git a/src/face-tracker-monitor.cpp b/src/face-tracker-monitor.cpp index 440cc70..c2a2390 100644 --- a/src/face-tracker-monitor.cpp +++ b/src/face-tracker-monitor.cpp @@ -3,6 +3,8 @@ #include "plugin-macros.generated.h" +#define MAX_ERROR 2 + struct face_tracker_monitor { obs_source_t *context; @@ -16,6 +18,8 @@ struct face_tracker_monitor obs_weak_source_t *source_ref; obs_weak_source_t *filter_ref; + + int n_error; }; static const char *ftmon_get_name(void *unused) @@ -55,14 +59,17 @@ static void ftmon_update(void *data, obs_data_t *settings) if (source_name && (!s->source_name || strcmp(source_name, s->source_name))) { bfree(s->source_name); s->source_name = bstrdup(source_name); + s->n_error = 0; } if (!filter_name || !*filter_name) { bfree(s->filter_name); s->filter_name = NULL; + s->n_error = 0; } else if (!s->filter_name || strcmp(filter_name, s->filter_name)) { bfree(s->filter_name); s->filter_name = bstrdup(filter_name); + s->n_error = 0; } s->notrack = obs_data_get_bool(settings, "notrack"); @@ -168,10 +175,19 @@ static void ftmon_tick(void *data, float second) tick_source(s, s->filter_ref, s->filter_name, get_filter_by_name); if (source_specified && !s->source_ref) { - blog(LOG_INFO, "failed to get source \"%s\"", s->source_name); + if (s->n_error < MAX_ERROR) { + blog(LOG_INFO, "failed to get source \"%s\"", s->source_name); + s->n_error ++; + } } else if (filter_specified && !s->filter_ref) { - blog(LOG_INFO, "failed to get filter \"%s\"", s->filter_name); + if (s->n_error < MAX_ERROR) { + blog(LOG_INFO, "failed to get filter \"%s\"", s->filter_name); + s->n_error ++; + } + } + else { + s->n_error = 0; } } diff --git a/src/face-tracker-ptz.cpp b/src/face-tracker-ptz.cpp index a22f23c..1e8b71f 100644 --- a/src/face-tracker-ptz.cpp +++ b/src/face-tracker-ptz.cpp @@ -18,8 +18,6 @@ #endif #include "dummy-backend.hpp" -#define debug_track(fmt, ...) // blog(LOG_INFO, fmt, __VA_ARGS__) - #define PTZ_MAX_X 0x18 #define PTZ_MAX_Y 0x14 #define PTZ_MAX_Z 0x07 @@ -210,6 +208,10 @@ static void ftptz_update(void *data, obs_data_t *settings) s->debug_notrack = obs_data_get_bool(settings, "debug_notrack"); s->debug_always_show = obs_data_get_bool(settings, "debug_always_show"); + debug_data_open(&s->debug_data_tracker, &s->debug_data_tracker_last, settings, "debug_data_tracker"); + debug_data_open(&s->debug_data_error, &s->debug_data_error_last, settings, "debug_data_error"); + debug_data_open(&s->debug_data_control, &s->debug_data_control_last, settings, "debug_data_control"); + s->ptz_max_x = obs_data_get_int(settings, "ptz_max_x"); s->ptz_max_y = obs_data_get_int(settings, "ptz_max_y"); s->ptz_max_z = obs_data_get_int(settings, "ptz_max_z"); @@ -246,6 +248,8 @@ static void ftptz_update(void *data, obs_data_t *settings) } static void cb_render_info(void *data, calldata_t *cd); +static void cb_get_state(void *data, calldata_t *cd); +static void cb_set_state(void *data, calldata_t *cd); static void *ftptz_create(obs_data_t *settings, obs_source_t *context) { @@ -261,6 +265,8 @@ static void *ftptz_create(obs_data_t *settings, obs_source_t *context) proc_handler_t *ph = obs_source_get_proc_handler(context); proc_handler_add(ph, "void render_info()", cb_render_info, s); + proc_handler_add(ph, "void get_state()", cb_get_state, s); + proc_handler_add(ph, "void set_state()", cb_set_state, s); return s; } @@ -276,6 +282,15 @@ static void ftptz_destroy(void *data) delete s->ftm; bfree(s->ptz_type); + if (s->debug_data_tracker) + fclose(s->debug_data_tracker); + if (s->debug_data_error) + fclose(s->debug_data_error); + if (s->debug_data_control) + fclose(s->debug_data_control); + bfree(s->debug_data_tracker_last); + bfree(s->debug_data_error_last); + bfree(s->debug_data_control_last); bfree(s); } @@ -392,8 +407,6 @@ static obs_properties_t *ftptz_properties(void *data) { obs_properties_t *pp = obs_properties_create(); - obs_properties_add_bool(pp, "debug_faces", "Show face detection results"); - obs_properties_add_bool(pp, "debug_always_show", "Always show information (useful for demo)"); obs_property_t *p = obs_properties_add_list(pp, "ptz-type", obs_module_text("PTZ Type"), OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING); obs_property_list_add_string(p, obs_module_text("None"), "dummy"); obs_property_list_add_string(p, obs_module_text("through PTZ Controls"), "obsptz"); @@ -413,6 +426,21 @@ static obs_properties_t *ftptz_properties(void *data) obs_properties_add_group(props, "output", obs_module_text("Output"), OBS_GROUP_NORMAL, pp); } + { + obs_properties_t *pp = obs_properties_create(); + obs_properties_add_bool(pp, "debug_faces", "Show face detection results"); + obs_properties_add_bool(pp, "debug_always_show", "Always show information (useful for demo)"); +#ifdef ENABLE_DEBUG_DATA + obs_properties_add_path(pp, "debug_data_tracker", "Save correlation tracker data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_error", "Save calculated error data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_control", "Save control data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); +#endif + obs_properties_add_group(props, "debug", obs_module_text("Debugging"), OBS_GROUP_NORMAL, pp); + } + return props; } @@ -639,7 +667,13 @@ static void tick_filter(struct face_tracker_ptz *s, float second) else if (n > +u_max[i]) n = +u_max[i]; s->u[i] = n; } - debug_track("tick_filter: u: %d %d %d uf: %f %f %f", s->u[0], s->u[1], s->u[2], uf.v[0], uf.v[1], uf.v[2]); + + if (s->debug_data_control) { + fprintf(s->debug_data_control, "%f\t%f\t%f\t%f\t%d\t%d\t%d\n", + os_gettime_ns() * 1e-9, + uf.v[0], uf.v[1], uf.v[2], + s->u[0], s->u[1], s->u[2] ); + } } static void ftf_activate(void *data) @@ -730,6 +764,7 @@ static inline void calculate_error(struct face_tracker_ptz *s) auto &tracker_rects = s->ftm->tracker_rects; for (int i=0; iftm->landmark_detection_data) { pointf_s center = landmark_center(tracker_rects[i].landmark); @@ -742,14 +777,18 @@ static inline void calculate_error(struct face_tracker_ptz *s) r.v[2] = sqrtf(area * (float)(4.0f / M_PI)); } + if (s->debug_data_tracker) { + fprintf(s->debug_data_tracker, "%f\t%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + r.v[0], r.v[1], r.v[2], score ); + } + r.v[0] -= get_width(tracker_rects[i].crop_rect) * s->track_x; r.v[1] += get_height(tracker_rects[i].crop_rect) * s->track_y; r.v[2] /= s->track_z; f3 w (tracker_rects[i].crop_rect); - float score = tracker_rects[i].rect.score; f3 e = (r-w) * score; - debug_track("calculate_error: %d %f %f %f %f", i, e.v[0], e.v[1], e.v[2], score); if (score>0.0f && !isnan(e)) { e_tot += e; sc_tot += score; @@ -762,6 +801,12 @@ static inline void calculate_error(struct face_tracker_ptz *s) else s->detect_err = f3(0, 0, 0); s->face_found = found; + + if (s->debug_data_error) { + fprintf(s->debug_data_error, "%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + s->detect_err.v[0], s->detect_err.v[1], s->detect_err.v[2] ); + } } static struct obs_source_frame *ftptz_filter_video(void *data, struct obs_source_frame *frame) @@ -849,6 +894,23 @@ static void cb_render_info(void *data, calldata_t *cd) draw_frame_info(s, landmark_only); } +static void cb_get_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_ptz*)data; + calldata_set_bool(cd, "paused", s->is_paused); +} + +static void cb_set_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_ptz*)data; + calldata_get_bool(cd, "paused", &s->is_paused); + + bool reset = false; + calldata_get_bool(cd, "reset", &reset); + if (reset) + ftptz_reset_tracking(NULL, NULL, s); +} + extern "C" void register_face_tracker_ptz() { diff --git a/src/face-tracker-ptz.hpp b/src/face-tracker-ptz.hpp index 35eb7c7..bc65420 100644 --- a/src/face-tracker-ptz.hpp +++ b/src/face-tracker-ptz.hpp @@ -33,6 +33,12 @@ struct face_tracker_ptz bool debug_faces; bool debug_notrack; bool debug_always_show; + FILE *debug_data_tracker; + FILE *debug_data_error; + FILE *debug_data_control; + char *debug_data_tracker_last; + char *debug_data_error_last; + char *debug_data_control_last; char *ptz_type; int ptz_max_x, ptz_max_y, ptz_max_z; diff --git a/src/face-tracker.cpp b/src/face-tracker.cpp index b8fe66d..7db69a0 100644 --- a/src/face-tracker.cpp +++ b/src/face-tracker.cpp @@ -13,9 +13,6 @@ #include "face-tracker-manager.hpp" #include "source_list.h" -// #define debug_track(fmt, ...) blog(LOG_INFO, fmt, __VA_ARGS__) -#define debug_track(fmt, ...) - static gs_effect_t *effect_ft = NULL; static inline void scale_texture(struct face_tracker_filter *s, float scale); @@ -99,6 +96,10 @@ static void ftf_update(void *data, obs_data_t *settings) s->debug_faces = obs_data_get_bool(settings, "debug_faces"); s->debug_notrack = obs_data_get_bool(settings, "debug_notrack"); s->debug_always_show = obs_data_get_bool(settings, "debug_always_show"); + + debug_data_open(&s->debug_data_tracker, &s->debug_data_tracker_last, settings, "debug_data_tracker"); + debug_data_open(&s->debug_data_error, &s->debug_data_error_last, settings, "debug_data_error"); + debug_data_open(&s->debug_data_control, &s->debug_data_control_last, settings, "debug_data_control"); } static void fts_update(void *data, obs_data_t *settings) @@ -116,6 +117,8 @@ static void fts_update(void *data, obs_data_t *settings) static void cb_render_frame(void *data, calldata_t *cd); static void cb_render_info(void *data, calldata_t *cd); static void cb_get_target_size(void *data, calldata_t *cd); +static void cb_get_state(void *data, calldata_t *cd); +static void cb_set_state(void *data, calldata_t *cd); static void *ftf_create(obs_data_t *settings, obs_source_t *context) { @@ -143,6 +146,8 @@ static void *ftf_create(obs_data_t *settings, obs_source_t *context) proc_handler_add(ph, "void render_frame(bool notrack)", cb_render_frame, s); proc_handler_add(ph, "void render_info(bool notrack)", cb_render_info, s); proc_handler_add(ph, "void get_target_size(out int width, out int height)", cb_get_target_size, s); + proc_handler_add(ph, "void get_state()", cb_get_state, s); + proc_handler_add(ph, "void set_state()", cb_set_state, s); return s; } @@ -180,6 +185,15 @@ static void ftf_destroy(void *data) bfree(s->target_name); obs_weak_source_release(s->target_ref); + if (s->debug_data_tracker) + fclose(s->debug_data_tracker); + if (s->debug_data_error) + fclose(s->debug_data_error); + if (s->debug_data_control) + fclose(s->debug_data_control); + bfree(s->debug_data_tracker_last); + bfree(s->debug_data_error_last); + bfree(s->debug_data_control_last); bfree(s); } @@ -278,6 +292,14 @@ static obs_properties_t *ftf_properties(void *data) obs_properties_add_bool(pp, "debug_faces", "Show face detection results"); obs_properties_add_bool(pp, "debug_notrack", "Stop tracking faces"); obs_properties_add_bool(pp, "debug_always_show", "Always show information (useful for demo)"); +#ifdef ENABLE_DEBUG_DATA + obs_properties_add_path(pp, "debug_data_tracker", "Save correlation tracker data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_error", "Save calculated error data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); + obs_properties_add_path(pp, "debug_data_control", "Save control data to file", + OBS_PATH_FILE_SAVE, DEBUG_DATA_PATH_FILTER, NULL ); +#endif obs_properties_add_group(props, "debug", obs_module_text("Debugging"), OBS_GROUP_NORMAL, pp); } @@ -376,6 +398,12 @@ static void tick_filter(struct face_tracker_filter *s, float second) } } + if (s->debug_data_control) { + fprintf(s->debug_data_control, "%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + u.v[0], u.v[1], u.v[2] ); + } + s->ftm->crop_cur = f3_to_rectf(u, s->width_with_aspect, s->height_with_aspect); } @@ -610,6 +638,7 @@ static inline void calculate_error(struct face_tracker_filter *s) auto &tracker_rects = s->ftm->tracker_rects; for (int i=0; iftm->landmark_detection_data) { pointf_s center = landmark_center(tracker_rects[i].landmark); @@ -622,15 +651,19 @@ static inline void calculate_error(struct face_tracker_filter *s) r.v[2] = sqrtf(area * (float)(4.0f / M_PI)); } + if (s->debug_data_tracker) { + fprintf(s->debug_data_tracker, "%f\t%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + r.v[0], r.v[1], r.v[2], score ); + } + r.v[0] -= get_width(tracker_rects[i].crop_rect) * s->track_x; r.v[1] += get_height(tracker_rects[i].crop_rect) * s->track_y; r.v[2] /= s->track_z; r = ensure_range(r, s); f3 w (tracker_rects[i].crop_rect); - float score = tracker_rects[i].rect.score; f3 e = (r-w) * score; - debug_track("calculate_error: %d %f %f %f %f", i, e.v[0], e.v[1], e.v[2], score); if (score>0.0f && !isnan(e)) { e_tot += e; sc_tot += score; @@ -642,6 +675,12 @@ static inline void calculate_error(struct face_tracker_filter *s) s->detect_err = e_tot * (1.0f / sc_tot); else s->detect_err = f3(0, 0, 0); + + if (s->debug_data_error) { + fprintf(s->debug_data_error, "%f\t%f\t%f\t%f\n", + os_gettime_ns() * 1e-9, + s->detect_err.v[0], s->detect_err.v[1], s->detect_err.v[2] ); + } } static inline void draw_sprite_crop(float width, float height, float x0, float y0, float x1, float y1); @@ -651,10 +690,15 @@ static inline void scale_texture(struct face_tracker_filter *s, float scale) if (!s->texrender_scaled) s->texrender_scaled = gs_texrender_create(GS_R8, GS_ZS_NONE); const uint32_t cx = s->known_width / scale, cy = s->known_height / scale; + + const int align = 8; + uint32_t width = (cx + align - 1) / align * align; + uint32_t height = (cy + align - 1) / align * align; + gs_texrender_reset(s->texrender_scaled); gs_blend_state_push(); gs_blend_function(GS_BLEND_ONE, GS_BLEND_ZERO); - if (gs_texrender_begin(s->texrender_scaled, cx, cy)) { + if (gs_texrender_begin(s->texrender_scaled, width, height)) { gs_ortho(0.0f, (float)cx, 0.0f, (float)cy, -100.0f, 100.0f); gs_texture_t *tex = gs_texrender_get_texture(s->texrender); if (tex && effect_ft) { @@ -675,6 +719,10 @@ static inline int stage_to_surface(struct face_tracker_filter *s, float scale) if (width<=0 || height<=0) return 1; + const int align = 8; + width = (width + align - 1) / align * align; + height = (height + align - 1) / align * align; + gs_texture_t *tex = gs_texrender_get_texture(s->texrender_scaled); if (!tex) return 2; @@ -931,6 +979,23 @@ static void cb_get_target_size(void *data, calldata_t *cd) calldata_set_int(cd, "height", (int)s->known_height); } +static void cb_get_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_filter*)data; + calldata_set_bool(cd, "paused", s->is_paused); +} + +static void cb_set_state(void *data, calldata_t *cd) +{ + auto *s = (struct face_tracker_filter*)data; + calldata_get_bool(cd, "paused", &s->is_paused); + + bool reset = false; + calldata_get_bool(cd, "reset", &reset); + if (reset) + ftf_reset_tracking(NULL, NULL, s); +} + extern "C" void register_face_tracker_filter() { diff --git a/src/face-tracker.hpp b/src/face-tracker.hpp index 379da3d..fdfb21b 100644 --- a/src/face-tracker.hpp +++ b/src/face-tracker.hpp @@ -43,6 +43,12 @@ struct face_tracker_filter bool debug_faces; bool debug_notrack; bool debug_always_show; + FILE *debug_data_tracker; + FILE *debug_data_error; + FILE *debug_data_control; + char *debug_data_tracker_last; + char *debug_data_error_last; + char *debug_data_control_last; bool is_paused; obs_hotkey_pair_id hotkey_pause; diff --git a/src/helper.cpp b/src/helper.cpp index 60200a0..16e35d7 100644 --- a/src/helper.cpp +++ b/src/helper.cpp @@ -135,3 +135,32 @@ void draw_landmark(const std::vector &landmark) gs_render_stop(GS_LINES); } + +void debug_data_open(FILE **dest, char **last_name, obs_data_t *settings, const char *name) +{ + const char *debug_data = obs_data_get_string(settings, name); + + // If the file name is not changed, just return. + if (*last_name && debug_data && strcmp(*last_name, debug_data) == 0) + return; + + // If both file names are empty, just return. + if (!*last_name && (!debug_data || !*debug_data)) + return; + + if (*dest) + fclose(*dest); + *dest = NULL; + + if (*last_name) + bfree(*last_name); + *last_name = NULL; + + if (debug_data && *debug_data) { + *dest = fopen(debug_data, "a"); + if (!*dest) { + blog(LOG_ERROR, "%s: Failed to open file \"%s\"", name, debug_data); + } + *last_name = bstrdup(debug_data); + } +} diff --git a/src/helper.hpp b/src/helper.hpp index 0838c39..e419e3c 100644 --- a/src/helper.hpp +++ b/src/helper.hpp @@ -2,6 +2,12 @@ #include #include +#ifdef _WIN32 +#define DEBUG_DATA_PATH_FILTER "TSV Files (*.tsv);;Data Files (*.dat);;All Files (*.*)" +#else +#define DEBUG_DATA_PATH_FILTER "Data Files (*.dat);;TSV Files (*.tsv);;All Files (*.*)" +#endif + struct pointf_s { float x; @@ -98,3 +104,5 @@ inline double from_dB(double x) { return exp(x * (M_LN10/20)); } + +void debug_data_open(FILE **dest, char **last_name, obs_data_t *settings, const char *name); diff --git a/src/module-main.c b/src/module-main.c index c379aa1..981cef4 100644 --- a/src/module-main.c +++ b/src/module-main.c @@ -1,5 +1,8 @@ #include #include "plugin-macros.generated.h" +#ifdef WITH_DOCK +#include "../ui/face-tracker-dock.hpp" +#endif // WITH_DOCK OBS_DECLARE_MODULE() OBS_MODULE_USE_DEFAULT_LOCALE(PLUGIN_NAME, "en-US") @@ -14,9 +17,15 @@ bool obs_module_load(void) register_face_tracker_filter(); register_face_tracker_ptz(); register_face_tracker_monitor(); +#ifdef WITH_DOCK + ft_docks_init(); +#endif // WITH_DOCK return true; } void obs_module_unload() { +#ifdef WITH_DOCK + ft_docks_release(); +#endif // WITH_DOCK } diff --git a/src/plugin-macros.h.in b/src/plugin-macros.h.in index d900f8f..e3ba3d0 100644 --- a/src/plugin-macros.h.in +++ b/src/plugin-macros.h.in @@ -24,6 +24,7 @@ with this program. If not, see #cmakedefine WITH_PTZ_TCP #cmakedefine ENABLE_MONITOR_USER +#cmakedefine WITH_DOCK #define blog(level, msg, ...) blog(level, "[" PLUGIN_NAME "] " msg, ##__VA_ARGS__) diff --git a/ui/face-tracker-dock-internal.hpp b/ui/face-tracker-dock-internal.hpp new file mode 100644 index 0000000..a53f55b --- /dev/null +++ b/ui/face-tracker-dock-internal.hpp @@ -0,0 +1,31 @@ +#pragma once +#include +#include +#include "util/threading.h" + +struct face_tracker_dock_s +{ + obs_display_t *disp; + pthread_mutex_t mutex; + volatile long ref; + + obs_source_t *src_monitor; +}; + +static inline void face_tracker_dock_addref(struct face_tracker_dock_s *data) +{ + if (!data) + return; + os_atomic_inc_long(&data->ref); +} + +struct face_tracker_dock_s *face_tracker_dock_create(); +void face_tracker_dock_destroy(struct face_tracker_dock_s *data); + +static inline void face_tracker_dock_release(struct face_tracker_dock_s *data) +{ + if (!data) + return; + if (os_atomic_dec_long(&data->ref) == 0) + face_tracker_dock_destroy(data); +} diff --git a/ui/face-tracker-dock.cpp b/ui/face-tracker-dock.cpp new file mode 100644 index 0000000..8f9e0be --- /dev/null +++ b/ui/face-tracker-dock.cpp @@ -0,0 +1,530 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "plugin-macros.generated.h" +#include "face-tracker-dock.hpp" +#include "face-tracker-widget.hpp" +#include "face-tracker-dock-internal.hpp" + +#define SAVE_DATA_NAME PLUGIN_NAME"-dock" +#define OBJ_NAME_SUFFIX "_ft_dock" + +void FTDock::closeEvent(QCloseEvent *event) +{ + QDockWidget::closeEvent(event); +} + +// accessed only from UI thread +static std::vector *docks; + +static std::string generate_unique_name() +{ + for (int n=0;;) { + char name[32] = "FTDock"; + if (n) + snprintf(name, sizeof(name), "FTDock-%d", n); + bool found = false; + if (docks) for (size_t i=0; isize(); i++) { + if ((*docks)[i]->name == name) + found = true; + } + if (!found) + return name; + } +} + +void ft_dock_add(const char *name, obs_data_t *props) +{ + auto *main_window = static_cast(obs_frontend_get_main_window()); + auto *dock = new FTDock(main_window); + dock->name = name ? name : generate_unique_name(); + dock->setObjectName(QString::fromUtf8(dock->name.c_str()) + OBJ_NAME_SUFFIX); + dock->setWindowTitle(dock->name.c_str()); + dock->setAllowedAreas(Qt::AllDockWidgetAreas); + + dock->load_properties(props); + + main_window->addDockWidget(Qt::BottomDockWidgetArea, dock); + dock->action = (QAction*)obs_frontend_add_dock(dock); + + if (docks) + docks->push_back(dock); +} + +struct init_target_selector_s +{ + QComboBox *q; + int index; + const char *source_name; + const char *filter_name; +}; + +static bool init_target_selector_compare_name(struct init_target_selector_s *ctx, const char *source_name, const char *filter_name) +{ + if (strcmp(ctx->source_name, source_name) != 0) + return false; + + // both expect source + if (!ctx->filter_name && !filter_name) + return true; + + // either one expects source + if (!ctx->filter_name || !filter_name) + return false; + + return strcmp(ctx->filter_name, filter_name) == 0; +} + +static void init_target_selector_cb_add(struct init_target_selector_s *ctx, obs_source_t *source, obs_source_t *filter) +{ + QString text; + QList val; + + const char *name = obs_source_get_name(source); + const char *filter_name = NULL; + text = QString::fromUtf8(name); + val.append(QVariant(name)); + + if (filter) { + filter_name = obs_source_get_name(filter); + text += " / "; + text += QString::fromUtf8(filter_name); + val.append(QVariant(filter_name)); + } + + if (ctx->index < ctx->q->count()) { + ctx->q->setItemText(ctx->index, text); + ctx->q->setItemData(ctx->index, QVariant(val)); + } + else + ctx->q->insertItem(ctx->index, text, QVariant(val)); + + if (ctx->source_name) { + if (init_target_selector_compare_name(ctx, name, filter_name)) + ctx->q->setCurrentIndex(ctx->index); + } + + ctx->index++; +} + +static void init_target_selector_cb_filter(obs_source_t *parent, obs_source_t *child, void *param) +{ + auto *ctx = (struct init_target_selector_s *)param; + + const char *id = obs_source_get_id(child); + if (!strcmp(id, "face_tracker_filter") || !strcmp(id, "face_tracker_ptz")) { + init_target_selector_cb_add(ctx, parent, child); + } +} + +static bool init_target_selector_cb_source(void *data, obs_source_t *source) +{ + auto *ctx = (struct init_target_selector_s *)data; + + const char *id = obs_source_get_id(source); + if (!strcmp(id, "face_tracker_source")) { + init_target_selector_cb_add(ctx, source, NULL); + return true; + } + + obs_source_enum_filters(source, init_target_selector_cb_filter, data); + + return true; +} + +static void init_target_selector(QComboBox *q, const char *source_name=NULL, const char *filter_name=NULL) +{ + QString current = q->currentText(); + + if (filter_name && !*filter_name) + filter_name = NULL; + + init_target_selector_s ctx = {q, 0, source_name, filter_name}; + obs_enum_scenes(init_target_selector_cb_source, &ctx); + obs_enum_sources(init_target_selector_cb_source, &ctx); + + while (q->count() > ctx.index) + q->removeItem(ctx.index); + + if (current.length() && !source_name) { + int ix = q->findText(current); + if (ix >= 0) + q->setCurrentIndex(ix); + } +} + +void FTDock::checkTargetSelector() +{ + init_target_selector(targetSelector); +} + +void FTDock::frontendEvent_cb(enum obs_frontend_event event, void *private_data) +{ + auto *dock = static_cast(private_data); + dock->frontendEvent(event); +} + +FTDock::FTDock(QWidget *parent) + : QDockWidget(parent) +{ + data = face_tracker_dock_create(); + + data->src_monitor = obs_source_create_private("face_tracker_monitor", "monitor", NULL); + + resize(256, 256); + setMinimumSize(128, 128); + setAttribute(Qt::WA_DeleteOnClose); + + mainLayout = new QVBoxLayout(this); + auto *dockWidgetContents = new QWidget; + dockWidgetContents->setObjectName(QStringLiteral("contextContainer")); + dockWidgetContents->setLayout(mainLayout); + + targetSelector = new QComboBox(this); + init_target_selector(targetSelector); + mainLayout->addWidget(targetSelector); + connect(targetSelector, &QComboBox::currentTextChanged, this, &FTDock::targetSelectorChanged); + + pauseButton = new QCheckBox(obs_module_text("Pause"), this); + mainLayout->addWidget(pauseButton); + connect(pauseButton, &QCheckBox::stateChanged, this, &FTDock::pauseButtonChanged); + + resetButton = new QPushButton(obs_module_text("Reset"), this); + mainLayout->addWidget(resetButton); + connect(resetButton, &QPushButton::clicked, this, &FTDock::resetButtonClicked); + +#ifdef HAVE_PROPERTY_BUTTON + propertyButton = new QPushButton(obs_module_text("Properties"), this); + mainLayout->addWidget(propertyButton); + connect(propertyButton, &QPushButton::clicked, this, &FTDock::propertyButtonClicked); +#endif + + ftWidget = new FTWidget(data, this); + mainLayout->addWidget(ftWidget); + + notrackButton = new QCheckBox(obs_module_text("Show all region"), this); + mainLayout->addWidget(notrackButton); + connect(notrackButton, &QCheckBox::stateChanged, this, &FTDock::notrackButtonChanged); + + setWidget(dockWidgetContents); + + connect(this, &FTDock::scenesMayChanged, this, &FTDock::checkTargetSelector); + updateState(); + + connect(this, &FTDock::dataChanged, this, &FTDock::updateWidget); + + obs_frontend_add_event_callback(frontendEvent_cb, this); +} + +FTDock::~FTDock() +{ + obs_frontend_remove_event_callback(frontendEvent_cb, this); + + face_tracker_dock_release(data); + if (action) + delete action; + if (docks) for (size_t i=0; isize(); i++) { + if ((*docks)[i] == this) { + docks->erase(docks->begin()+i); + break; + } + } +} + +void FTDock::showEvent(QShowEvent *event) +{ + blog(LOG_INFO, "FTDock::showEvent"); +} + +void FTDock::hideEvent(QHideEvent *event) +{ + blog(LOG_INFO, "FTDock::hideEvent"); +} + +void FTDock::frontendEvent(enum obs_frontend_event event) +{ +} + +void FTDock::targetSelectorChanged() +{ + if (updating_widget) + return; + + updateState(); +} + +OBSSource FTDock::get_source() +{ + OBSSource target; + QList data = targetSelector->currentData().toList(); + + for (int i=0; i &target_data) +{ + blog(LOG_INFO, "set_monitor monitor=%p", monitor); + OBSData data = obs_data_create(); + obs_data_release(data); + + if (target_data.count() < 1) + return; + + obs_data_set_string(data, "source_name", target_data[0].toByteArray().constData()); + + obs_data_set_string(data, "filter_name", + target_data.count() > 1 ? target_data[1].toByteArray().constData() : ""); + + obs_source_update(monitor, data); +} + +void FTDock::updateState() +{ + OBSSource target = get_source(); + proc_handler_t *ph = obs_source_get_proc_handler(target); + if (!ph) + return; + + calldata_t cd = {0}; + if (proc_handler_call(ph, "get_state", &cd)) { + bool b; + + if (calldata_get_bool(&cd, "paused", &b)) { + pauseButton->setCheckState(b ? Qt::Checked : Qt::Unchecked); + } + } + calldata_free(&cd); + + if (!data) + return; + + pthread_mutex_lock(&data->mutex); + + if (data->src_monitor) + set_monitor(data->src_monitor, targetSelector->currentData().toList()); + + pthread_mutex_unlock(&data->mutex); +} + +void FTDock::updateWidget() +{ + if (updating_widget) + return; + updating_widget = true; + pthread_mutex_lock(&data->mutex); + + if (data->src_monitor) { + obs_data_t *props = obs_source_get_settings(data->src_monitor); + + const char *source_name = obs_data_get_string(props, "source_name"); + const char *filter_name = obs_data_get_string(props, "filter_name"); + init_target_selector(targetSelector, source_name, filter_name); + + bool notrack = obs_data_get_bool(props, "notrack"); + notrackButton->setCheckState(notrack ? Qt::Checked : Qt::Unchecked); + + obs_data_release(props); + } + + pthread_mutex_unlock(&data->mutex); + updating_widget = false; +} + +void FTDock::pauseButtonChanged(int state) +{ + OBSSource target = get_source(); + proc_handler_t *ph = obs_source_get_proc_handler(target); + if (!ph) + return; + + calldata_t cd = {0}; + calldata_set_bool(&cd, "paused", state==Qt::Checked); + proc_handler_call(ph, "set_state", &cd); + calldata_free(&cd); +} + +void FTDock::resetButtonClicked(bool checked) +{ + UNUSED_PARAMETER(checked); + + OBSSource target = get_source(); + proc_handler_t *ph = obs_source_get_proc_handler(target); + if (!ph) + return; + + calldata_t cd = {0}; + calldata_set_bool(&cd, "reset", true); + proc_handler_call(ph, "set_state", &cd); + calldata_free(&cd); +} + +#ifdef HAVE_PROPERTY_BUTTON +void FTDock::propertyButtonClicked(bool checked) +{ + UNUSED_PARAMETER(checked); + + QList data = targetSelector->currentData().toList(); + if (data.count() < 1) + return; + + const char *name = data[0].toByteArray().constData(); + + obs_source_t *target = obs_get_source_by_name(name); + + if (data.count() == 1) + obs_frontend_open_source_properties(target); + else + obs_frontend_open_source_filters(target); + + obs_source_release(target); +} +#endif // HAVE_PROPERTY_BUTTON + +void FTDock::notrackButtonChanged(int state) +{ + if (!data || !data->src_monitor) + return; + obs_data_t *props = obs_data_create(); + obs_data_set_bool(props, "notrack", state==Qt::Checked); + obs_source_update(data->src_monitor, props); + obs_data_release(props); +} + +static void save_load_ft_docks(obs_data_t *save_data, bool saving, void *) +{ + blog(LOG_INFO, "save_load_ft_docks saving=%d", (int)saving); + if (!docks) + return; + if (saving) { + obs_data_t *props = obs_data_create(); + obs_data_array_t *array = obs_data_array_create(); + for (size_t i=0; isize(); i++) { + FTDock *d = (*docks)[i]; + obs_data_t *obj = obs_data_create(); + d->save_properties(obj); + obs_data_set_string(obj, "name", d->name.c_str()); + obs_data_array_push_back(array, obj); + obs_data_release(obj); + } + obs_data_set_array(props, "docks", array); + obs_data_set_obj(save_data, SAVE_DATA_NAME, props); + obs_data_array_release(array); + obs_data_release(props); + } + + else /* loading */ { + if (docks) while (docks->size()) { + (*docks)[docks->size()-1]->close(); + delete (*docks)[docks->size()-1]; + } + + obs_data_t *props = obs_data_get_obj(save_data, SAVE_DATA_NAME); + if (!props) { + blog(LOG_INFO, "save_load_ft_docks: creating default properties"); + props = obs_data_create(); + } + + obs_data_array_t *array = obs_data_get_array(props, "docks"); + size_t count = obs_data_array_count(array); + for (size_t i=0; i; + obs_frontend_add_save_callback(save_load_ft_docks, NULL); + + QAction *action = static_cast(obs_frontend_add_tools_menu_qaction( + obs_module_text("New Face Tracker Dock...") )); + blog(LOG_INFO, "ft_docks_init: Adding face tracker dock menu action=%p", action); + auto cb = [] { + obs_data_t *props = obs_data_create(); + FTDock::default_properties(props); + ft_dock_add(NULL, props); + obs_data_release(props); + }; + QAction::connect(action, &QAction::triggered, cb); +} + +void ft_docks_release() +{ + delete docks; + docks = NULL; +} + +void FTDock::default_properties(obs_data_t *props) +{ +} + +void FTDock::save_properties(obs_data_t *props) +{ + // Save indicates a source or a filter has been changed. + scenesMayChanged(); + + pthread_mutex_lock(&data->mutex); + + obs_data_t *prop = obs_source_get_settings(data->src_monitor); + if (prop) { + obs_data_set_obj(props, "monitor", prop); + obs_data_release(prop); + } + + pthread_mutex_unlock(&data->mutex); +} + +void FTDock::load_properties(obs_data_t *props) +{ + pthread_mutex_lock(&data->mutex); + + if (data && data->src_monitor) { + obs_data_t *prop = obs_data_get_obj(props, "monitor"); + if (prop) { + obs_source_update(data->src_monitor, prop); + obs_data_release(prop); + } + } + + pthread_mutex_unlock(&data->mutex); + + dataChanged(); +} + +struct face_tracker_dock_s *face_tracker_dock_create() +{ + struct face_tracker_dock_s *data = (struct face_tracker_dock_s *)bzalloc(sizeof(struct face_tracker_dock_s)); + data->ref = 1; + pthread_mutex_init(&data->mutex, NULL); + return data; +} + +void face_tracker_dock_destroy(struct face_tracker_dock_s *data) +{ + obs_display_destroy(data->disp); + data->disp = NULL; + obs_source_release(data->src_monitor); + pthread_mutex_destroy(&data->mutex); + bfree(data); +} diff --git a/ui/face-tracker-dock.hpp b/ui/face-tracker-dock.hpp new file mode 100644 index 0000000..83fe56b --- /dev/null +++ b/ui/face-tracker-dock.hpp @@ -0,0 +1,83 @@ +#pragma once + +#ifdef __cplusplus +#include +#include +#include +#include +#include +#include +#include + +// the necessary APIs are implemented at 27.0.1-107-g5b18faeb4. +#if LIBOBS_API_VER > MAKE_SEMANTIC_VERSION(27, 0, 1) +#define HAVE_PROPERTY_BUTTON +#endif + +class FTDock : public QDockWidget { + Q_OBJECT + +public: + class FTWidget *widget; + std::string name; + QPointer action = 0; + struct face_tracker_dock_s *data; + + class QVBoxLayout *mainLayout; + class QComboBox *targetSelector; + class QCheckBox *pauseButton; + class QPushButton *resetButton; +#ifdef HAVE_PROPERTY_BUTTON + class QPushButton *propertyButton; +#endif + class FTWidget *ftWidget; + class QCheckBox *notrackButton; + + bool updating_widget = false; + +public: + FTDock(QWidget *parent = nullptr); + ~FTDock(); + void closeEvent(QCloseEvent *event) override; + + static void default_properties(obs_data_t *); + void save_properties(obs_data_t*); + void load_properties(obs_data_t*); + +private: + void showEvent(QShowEvent *event) override; + void hideEvent(QHideEvent *event) override; + + void frontendEvent(enum obs_frontend_event event); + static void frontendEvent_cb(enum obs_frontend_event event, void *private_data); + + OBSSource get_source(); + +signals: + void scenesMayChanged(); + void dataChanged(); + +public slots: + void checkTargetSelector(); + void updateState(); + void updateWidget(); + +private slots: + void targetSelectorChanged(); + void pauseButtonChanged(int state); + void resetButtonClicked(bool checked); +#ifdef HAVE_PROPERTY_BUTTON + void propertyButtonClicked(bool checked); +#endif + void notrackButtonChanged(int state); +}; + +extern "C" { +#endif // __cplusplus +void ft_dock_add(const char *name, obs_data_t *props); +void ft_docks_init(); +void ft_docks_release(); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/ui/face-tracker-widget.cpp b/ui/face-tracker-widget.cpp new file mode 100644 index 0000000..c8b08bf --- /dev/null +++ b/ui/face-tracker-widget.cpp @@ -0,0 +1,316 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "plugin-macros.generated.h" +#include "face-tracker-widget.hpp" +#include "face-tracker-dock-internal.hpp" +#include "obsgui-helper.hpp" + + +static void draw(void *param, uint32_t cx, uint32_t cy) +{ + auto *data = (struct face_tracker_dock_s *)param; + + if (pthread_mutex_trylock(&data->mutex)) + return; + + gs_blend_state_push(); + gs_reset_blend_state(); + + if (data->src_monitor) { + int w_src = obs_source_get_width(data->src_monitor); + int h_src = obs_source_get_height(data->src_monitor); + if (w_src <= 0 || h_src <= 0) + goto err; + int w, h; + if (w_src * cy > h_src * cx) { + w = cx; + h = cx * h_src / w_src; + } else { + h = cy; + w = cy * w_src / h_src; + } + + gs_projection_push(); + gs_viewport_push(); + + gs_set_viewport((cx-w)*0.5, (cy-h)*0.5, w, h); + gs_ortho(0.0f, w_src, -1.0f, h_src, -100.0f, 100.0f); + + obs_source_video_render(data->src_monitor); + + gs_viewport_pop(); + gs_projection_pop(); + } +err: + + gs_blend_state_pop(); + + pthread_mutex_unlock(&data->mutex); +} + +FTWidget::FTWidget(struct face_tracker_dock_s *data_, QWidget *parent) + : QWidget(parent) + , eventFilter(BuildEventFilter()) +{ + face_tracker_dock_addref((data = data_)); + setAttribute(Qt::WA_PaintOnScreen); + setAttribute(Qt::WA_StaticContents); + setAttribute(Qt::WA_NoSystemBackground); + setAttribute(Qt::WA_OpaquePaintEvent); + setAttribute(Qt::WA_DontCreateNativeAncestors); + setAttribute(Qt::WA_NativeWindow); + + setMouseTracking(true); + QObject::installEventFilter(eventFilter.get()); +} + +FTWidget::~FTWidget() +{ + removeEventFilter(eventFilter.get()); + face_tracker_dock_release(data); +} + +OBSEventFilter *FTWidget::BuildEventFilter() +{ + return new OBSEventFilter([this](QObject *obj, QEvent *event) { + UNUSED_PARAMETER(obj); + + switch (event->type()) { + case QEvent::MouseButtonPress: + case QEvent::MouseButtonRelease: + case QEvent::MouseButtonDblClick: + return this->HandleMouseClickEvent( + static_cast(event)); + case QEvent::MouseMove: + case QEvent::Enter: + case QEvent::Leave: + return this->HandleMouseMoveEvent( + static_cast(event)); + + case QEvent::Wheel: + return this->HandleMouseWheelEvent( + static_cast(event)); + case QEvent::KeyPress: + case QEvent::KeyRelease: + return this->HandleKeyEvent( + static_cast(event)); + default: + return false; + } + }); +} + +void FTWidget::CreateDisplay() +{ + if (!data) + return; + if (data->disp || !windowHandle()->isExposed()) + return; + + blog(LOG_INFO, "FTWidget::CreateDisplay %p", this); + + QSize size = GetPixelSize(this); + gs_init_data info = {}; + info.cx = size.width(); + info.cy = size.height(); + info.format = GS_BGRA; + info.zsformat = GS_ZS_NONE; + QWindow *window = windowHandle(); + if (!window) { + blog(LOG_ERROR, "FTWidget %p: windowHandle() returns NULL", this); + return; + } + if (!QTToGSWindow(window, info.window)) { + blog(LOG_ERROR, "FTWidget %p: QTToGSWindow failed", this); + return; + } + data->disp = obs_display_create(&info, 0); + obs_display_add_draw_callback(data->disp, draw, data); +} + +void FTWidget::resizeEvent(QResizeEvent *event) +{ + QWidget::resizeEvent(event); + CreateDisplay(); + + QSize size = GetPixelSize(this); + obs_display_resize(data->disp, size.width(), size.height()); +} + +void FTWidget::paintEvent(QPaintEvent *event) +{ + CreateDisplay(); +} + +class QPaintEngine *FTWidget::paintEngine() const +{ + return NULL; +} + +void FTWidget::closeEvent(QCloseEvent *event) +{ + setShown(false); +} + +void FTWidget::setShown(bool shown) +{ + if (shown && !data->disp) { + CreateDisplay(); + } + if (!shown && data->disp) { + obs_display_destroy(data->disp); + data->disp = NULL; + } +} + +#define INTERACT_KEEP_SOURCE (1<<30) + +static int TranslateQtKeyboardEventModifiers(QInputEvent *event, + bool mouseEvent) +{ + int obsModifiers = INTERACT_NONE; + + if (event->modifiers().testFlag(Qt::ShiftModifier)) + obsModifiers |= INTERACT_SHIFT_KEY; + if (event->modifiers().testFlag(Qt::AltModifier)) + obsModifiers |= INTERACT_ALT_KEY; +#ifdef __APPLE__ + // Mac: Meta = Control, Control = Command + if (event->modifiers().testFlag(Qt::ControlModifier)) + obsModifiers |= INTERACT_COMMAND_KEY; + if (event->modifiers().testFlag(Qt::MetaModifier)) + obsModifiers |= INTERACT_CONTROL_KEY; +#else + // Handle windows key? Can a browser even trap that key? + if (event->modifiers().testFlag(Qt::ControlModifier)) + obsModifiers |= INTERACT_CONTROL_KEY; +#endif + + if (!mouseEvent) { + if (event->modifiers().testFlag(Qt::KeypadModifier)) + obsModifiers |= INTERACT_IS_KEY_PAD; + } + + return obsModifiers; +} + +static int TranslateQtMouseEventModifiers(QMouseEvent *event) +{ + int modifiers = TranslateQtKeyboardEventModifiers(event, true); + + if (event->buttons().testFlag(Qt::LeftButton)) + modifiers |= INTERACT_MOUSE_LEFT; + if (event->buttons().testFlag(Qt::MiddleButton)) + modifiers |= INTERACT_MOUSE_MIDDLE; + if (event->buttons().testFlag(Qt::RightButton)) + modifiers |= INTERACT_MOUSE_RIGHT; + + return modifiers; +} + +bool FTWidget::HandleMouseClickEvent(QMouseEvent *event) +{ + bool mouseUp = event->type() == QEvent::MouseButtonRelease; + int clickCount = 1; + if (event->type() == QEvent::MouseButtonDblClick) + clickCount = 2; + + struct obs_mouse_event mouseEvent = {}; + + mouseEvent.modifiers = TranslateQtMouseEventModifiers(event); + + int32_t button = 0; + + switch (event->button()) { + case Qt::LeftButton: + button = MOUSE_LEFT; + if (mouseUp) mouseEvent.modifiers |= INTERACT_KEEP_SOURCE; // Not to change i_mouse if released outside + break; + case Qt::MiddleButton: + button = MOUSE_MIDDLE; + break; + case Qt::RightButton: + button = MOUSE_RIGHT; + break; + default: + blog(LOG_WARNING, "unknown button type %d", event->button()); + return false; + } + + return true; +} + +bool FTWidget::HandleMouseMoveEvent(QMouseEvent *event) +{ + struct obs_mouse_event mouseEvent = {}; + + bool mouseLeave = event->type() == QEvent::Leave; + + if (!mouseLeave) + mouseEvent.modifiers = TranslateQtMouseEventModifiers(event); + + int x = event->x(); + int y = event->y(); + + return true; +} + +bool FTWidget::HandleMouseWheelEvent(QWheelEvent *event) +{ + struct obs_mouse_event mouseEvent = {}; + + mouseEvent.modifiers = TranslateQtKeyboardEventModifiers(event, true); + + int xDelta = 0; + int yDelta = 0; + + const QPoint angleDelta = event->angleDelta(); + if (!event->pixelDelta().isNull()) { + if (angleDelta.x()) + xDelta = event->pixelDelta().x(); + else + yDelta = event->pixelDelta().y(); + } else { + if (angleDelta.x()) + xDelta = angleDelta.x(); + else + yDelta = angleDelta.y(); + } + +#if QT_VERSION >= QT_VERSION_CHECK(5, 14, 0) + const QPointF position = event->position(); + const int x = position.x(); + const int y = position.y(); +#else + const int x = event->x(); + const int y = event->y(); +#endif + + return true; +} + +bool FTWidget::HandleKeyEvent(QKeyEvent *event) +{ + struct obs_key_event keyEvent; + + QByteArray text = event->text().toUtf8(); + keyEvent.modifiers = TranslateQtKeyboardEventModifiers(event, false); + keyEvent.text = text.data(); + keyEvent.native_modifiers = event->nativeModifiers(); + keyEvent.native_scancode = event->nativeScanCode(); + keyEvent.native_vkey = event->nativeVirtualKey(); + + bool keyUp = event->type() == QEvent::KeyRelease; + + // TODO: implement me obs_source_send_key_click(source, &keyEvent, keyUp); + + return true; +} diff --git a/ui/face-tracker-widget.hpp b/ui/face-tracker-widget.hpp new file mode 100644 index 0000000..e0a02da --- /dev/null +++ b/ui/face-tracker-widget.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +#define SCOPE_WIDGET_N_SRC 4 + +class OBSEventFilter; + +class FTWidget : public QWidget { + Q_OBJECT + + std::unique_ptr eventFilter; + + void CreateDisplay(); + void resizeEvent(QResizeEvent *event) override; + void paintEvent(QPaintEvent *event) override; + class QPaintEngine *paintEngine() const override; + void closeEvent(QCloseEvent *event) override; + + // for interactions + bool HandleMouseClickEvent(QMouseEvent *event); + bool HandleMouseMoveEvent(QMouseEvent *event); + bool HandleMouseWheelEvent(QWheelEvent *event); + bool HandleKeyEvent(QKeyEvent *event); + OBSEventFilter *BuildEventFilter(); + +public: + FTWidget(struct face_tracker_dock_s *data, QWidget *parent); + ~FTWidget(); + void setShown(bool shown); + +private: + struct face_tracker_dock_s *data; +}; + +typedef std::function EventFilterFunc; + +class OBSEventFilter : public QObject +{ + Q_OBJECT +public: + OBSEventFilter(EventFilterFunc filter_) : filter(filter_) {} + +protected: + bool eventFilter(QObject *obj, QEvent *event) + { + return filter(obj, event); + } + +public: + EventFilterFunc filter; +}; diff --git a/ui/obsgui-helper.hpp b/ui/obsgui-helper.hpp new file mode 100644 index 0000000..c691a6c --- /dev/null +++ b/ui/obsgui-helper.hpp @@ -0,0 +1,46 @@ +#pragma once +#include +#if !defined(_WIN32) && !defined(__APPLE__) // if Linux +#include +#include +#include +#endif + +// copied from obs-studio/UI/qt-wrappers.cpp and modified to support OBS-26 +static inline +bool QTToGSWindow(QWindow *window, gs_window &gswindow) +{ + bool success = true; + +#ifdef _WIN32 + gswindow.hwnd = (HWND)window->winId(); +#elif __APPLE__ + gswindow.view = (id)window->winId(); +#else +#ifdef ENABLE_WAYLAND + switch (obs_get_nix_platform()) { + case OBS_NIX_PLATFORM_X11_GLX: + case OBS_NIX_PLATFORM_X11_EGL: +#endif // ENABLE_WAYLAND + gswindow.id = window->winId(); + gswindow.display = obs_get_nix_platform_display(); +#ifdef ENABLE_WAYLAND + break; + case OBS_NIX_PLATFORM_WAYLAND: + QPlatformNativeInterface *native = + QGuiApplication::platformNativeInterface(); + gswindow.display = + native->nativeResourceForWindow("surface", window); + success = gswindow.display != nullptr; + break; + } +#endif // ENABLE_WAYLAND +#endif + return success; +} + +// copied from obs-studio/UI/display-helpers.hpp +static inline QSize GetPixelSize(QWidget *widget) +{ + return widget->size() * widget->devicePixelRatioF(); +}