From a84eea57c868d61a4498b727be20d092c9d86939 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Fri, 19 Jan 2018 08:58:37 +0000 Subject: [PATCH] Finishing HOG recording, starting to integrate a visualizer, so that output videos/images are consistent in C++ and C# --- gui/OpenFaceOffline/MainWindow.xaml.cs | 20 ++- lib/local/CppInerop/CppInerop.vcxproj | 1 + lib/local/CppInerop/CppInerop.vcxproj.filters | 3 + lib/local/CppInerop/CppInterop.cpp | 3 +- lib/local/CppInerop/FaceAnalyserInterop.h | 22 ++- lib/local/CppInerop/VisualizerInterop.h | 136 ++++++++++++++++++ 6 files changed, 176 insertions(+), 9 deletions(-) create mode 100644 lib/local/CppInerop/VisualizerInterop.h diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index 0c6e554..df01294 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -108,7 +108,10 @@ namespace OpenFaceOffline FaceAnalyserManaged face_analyser; GazeAnalyserManaged gaze_analyser; - // Recording parameters (default values) + // For visualization of results + Visualizer visualizer_of; + + // For output recording Recorder recorder; public bool RecordAligned { get; set; } = false; // Aligned face images @@ -300,12 +303,12 @@ namespace OpenFaceOffline // Predic eye gaze gaze_analyser.AddNextFrame(clnf_model, detectionSucceeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); + // Only the final face will contain the details + VisualizeFeatures(frame, landmarks, i == 0, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); + // Record an observation RecordObservation(recorder, detectionSucceeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); - // Only the final face will contain the details - VisualizeFeatures(frame, landmarks, i==0, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); - } latest_img = null; @@ -468,6 +471,7 @@ namespace OpenFaceOffline private void VisualizeFeatures(RawImage frame, List> landmarks, bool new_image, double fx, double fy, double cx, double cy, double progress) { + List> lines = null; List> eye_landmarks = null; List> gaze_lines = null; @@ -486,6 +490,14 @@ namespace OpenFaceOffline double scale = 0; + // Helps with recording and showing the visualizations + ///visualizer.SetObservationFaceAlign(sim_warped_img); + //visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); + //visualizer.SetObservationLandmarks(face_model.detected_landmarks, 1.0, face_model.detection_success); // Set confidence to high to make sure we always visualize + //visualizer.SetObservationPose(pose_estimate, 1.0); + //visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy), face_model.detection_certainty); + + if (detectionSucceeding) { diff --git a/lib/local/CppInerop/CppInerop.vcxproj b/lib/local/CppInerop/CppInerop.vcxproj index 3a8b7d9..c1d5fcc 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj +++ b/lib/local/CppInerop/CppInerop.vcxproj @@ -186,6 +186,7 @@ + diff --git a/lib/local/CppInerop/CppInerop.vcxproj.filters b/lib/local/CppInerop/CppInerop.vcxproj.filters index 8860193..dd35bd0 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj.filters +++ b/lib/local/CppInerop/CppInerop.vcxproj.filters @@ -47,5 +47,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/lib/local/CppInerop/CppInterop.cpp b/lib/local/CppInerop/CppInterop.cpp index 3dfc6a9..f60fcc0 100644 --- a/lib/local/CppInerop/CppInterop.cpp +++ b/lib/local/CppInerop/CppInterop.cpp @@ -40,4 +40,5 @@ #include "CameraInterop.h" #include "ImageReader.h" #include "FaceDetectorInterop.h" -#include "RecorderInterop.h" \ No newline at end of file +#include "RecorderInterop.h" +#include "VisualizerInterop.h" \ No newline at end of file diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index 8d54caa..089663d 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -335,14 +335,28 @@ public: return HOG_vis_image; } - OpenCVWrappers::RawImage^ GetLatestHOGFeature(System::Int32^ num_rows, System::Int32^ num_cols, System::Int32^ num_channels) { - num_rows = gcnew System::Int32(*this->num_rows); - num_cols = gcnew System::Int32(*this->num_cols); - num_channels = gcnew System::Int32(31); + OpenCVWrappers::RawImage^ GetLatestHOGFeature() { OpenCVWrappers::RawImage^ HOG_feature = gcnew OpenCVWrappers::RawImage(*hog_features); return HOG_feature; } + // As the number of HOG rows and columns might not be known in advance, have methods for querying them + int GetHOGRows() + { + return *num_rows; + } + + int GetHOGCols() + { + return *num_cols; + } + + // The number of channels is always the same + int GetHOGChannels() + { + return 31; + } + void Reset() { face_analyser->Reset(); diff --git a/lib/local/CppInerop/VisualizerInterop.h b/lib/local/CppInerop/VisualizerInterop.h new file mode 100644 index 0000000..4a19599 --- /dev/null +++ b/lib/local/CppInerop/VisualizerInterop.h @@ -0,0 +1,136 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt + +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#pragma unmanaged + +// Include all the unmanaged things we need. + +#include "Visualizer.h" + +#pragma managed + +using System::Collections::Generic::List; + +namespace UtilitiesOF { + + public ref class Visualizer + { + private: + + // OpenCV based video capture for reading from files + Utilities::Visualizer* m_visualizer; + + public: + + Visualizer(bool vis_track, bool vis_hog, bool vis_aligned) + { + m_visualizer = new Utilities::Visualizer(vis_track, vis_hog, vis_aligned); + } + + void SetObservationGaze(System::Tuple^ gaze_direction0, System::Tuple^ gaze_direction1, + List^>^ landmarks_2D, List^>^ landmarks_3D, + double confidence) + { + cv::Point3f gaze_direction0_cv(gaze_direction0->Item1, gaze_direction0->Item2, gaze_direction0->Item3); + cv::Point3f gaze_direction1_cv(gaze_direction1->Item1, gaze_direction1->Item2, gaze_direction1->Item3); + + // Construct an OpenCV matrix from the landmarks + std::vector landmarks_2D_cv; + for (int i = 0; i < landmarks_2D->Count; ++i) + { + landmarks_2D_cv.push_back(cv::Point2d(landmarks_2D[i]->Item1, landmarks_2D[i]->Item2)); + } + + // Construct an OpenCV matrix from the landmarks + std::vector landmarks_3D_cv; + for (int i = 0; i < landmarks_3D->Count; ++i) + { + landmarks_3D_cv.push_back(cv::Point3d(landmarks_3D[i]->Item1, landmarks_3D[i]->Item2, landmarks_3D[i]->Item3)); + } + + m_visualizer->SetObservationGaze(gaze_direction0_cv, gaze_direction1_cv, landmarks_2D_cv, landmarks_3D_cv, confidence); + } + + // Setting the observations + void SetObservationPose(List^ pose, double confidence) + { + cv::Vec6d pose_vec(pose[0], pose[1], pose[2], pose[3], pose[4], pose[5]); + m_visualizer->SetObservationPose(pose_vec, confidence); + } + + void SetObservationFaceAlign(OpenCVWrappers::RawImage^ aligned_face_image) + { + m_visualizer->SetObservationFaceAlign(aligned_face_image->Mat); + } + + void SetObservationHOG(bool success, OpenCVWrappers::RawImage^ observation_HOG, int num_cols, int num_rows) + { + m_visualizer->SetObservationHOG(observation_HOG->Mat, num_cols, num_rows); + } + + void SetObservationLandmarks(List^>^ landmarks_2D, double confidence, bool success) + { + // Construct an OpenCV matrix from the landmarks + cv::Mat_ landmarks_2D_mat(landmarks_2D->Count * 2, 1, 0.0); + for (int i = 0; i < landmarks_2D->Count; ++i) + { + landmarks_2D_mat.at(i, 0) = landmarks_2D[i]->Item1; + landmarks_2D_mat.at(i + landmarks_2D->Count, 0) = landmarks_2D[i]->Item2; + } + // TODO add visibilities + m_visualizer->SetObservationLandmarks(landmarks_2D_mat, confidence, success); + } + + // Finalizer. Definitely called before Garbage Collection, + // but not automatically called on explicit Dispose(). + // May be called multiple times. + !Visualizer() + { + // Automatically closes capture object before freeing memory. + if (m_visualizer != nullptr) + { + delete m_visualizer; + } + + } + + // Destructor. Called on explicit Dispose() only. + ~Visualizer() + { + this->!Visualizer(); + } + }; + +}