From dcb9d90eca2b7257c2e95a151a9168b32d385b7d Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Thu, 16 Nov 2017 17:46:34 +0000 Subject: [PATCH] Fixing recorder and visualizer for FaceLandmarkImg --- exe/FaceLandmarkImg/FaceLandmarkImg.cpp | 7 ++--- lib/local/FaceAnalyser/include/FaceAnalyser.h | 5 ++-- lib/local/FaceAnalyser/src/FaceAnalyser.cpp | 14 +++++++--- lib/local/Utilities/src/Visualizer.cpp | 27 +++++++++++++++++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 51c24f9..c103d74 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -248,7 +248,7 @@ int main (int argc, char **argv) // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { - face_analyser.PredictStaticAUs(captured_image, face_model.detected_landmarks); + face_analyser.PredictStaticAUsAndComputeFeatures(captured_image, face_model.detected_landmarks); face_analyser.GetLatestAlignedFace(sim_warped_img); face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } @@ -271,11 +271,12 @@ int main (int argc, char **argv) open_face_rec.SetObservationFaceAlign(sim_warped_img); open_face_rec.WriteObservation(); - // Grabbing the next frame in the sequence - captured_image = image_reader.GetNextImage(); } visualizer.ShowObservation(); + // Grabbing the next frame in the sequence + captured_image = image_reader.GetNextImage(); + } return 0; diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index c24f818..4a66d88 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -73,9 +73,8 @@ public: std::vector> GetCurrentAUsReg() const; // AU intensity std::vector> GetCurrentAUsCombined() const; // Both presense and intensity - // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity - // This call is useful for detecting action units in images - std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks); + // A standalone call for predicting AUs and computing face texture features from a static image + void PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, const cv::Mat_& detected_landmarks); void Reset(); diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index 2627100..c7cbd61 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -248,7 +248,7 @@ int GetViewId(const vector orientations_all, const cv::Vec3d& orienta } -std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks) +void FaceAnalyser::PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, const cv::Mat_& detected_landmarks) { // Extract shape parameters from the detected landmarks @@ -259,6 +259,16 @@ std::pair>, std::vector hog_descriptor; Extract_FHOG_descriptor(hog_descriptor, aligned_face_for_au, this->num_hog_rows, this->num_hog_cols); @@ -303,8 +313,6 @@ std::pair>, std::vector>, std::vector>>(AU_predictions_intensity, AU_predictions_occurence); - } void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online) diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index 2b65454..5448cb7 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -82,6 +82,7 @@ Visualizer::Visualizer(bool vis_track, bool vis_hog, bool vis_align) this->vis_align = vis_align; } +// Setting the image on which to draw void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy) { captured_image = canvas.clone(); @@ -89,20 +90,42 @@ void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, f this->fy = fy; this->cx = cx; this->cy = cy; + + // Clearing other images + hog_image = cv::Mat(); + aligned_face_image = cv::Mat(); + } void Visualizer::SetObservationFaceAlign(const cv::Mat& aligned_face) { - this->aligned_face_image = aligned_face; + if(this->aligned_face_image.empty()) + { + this->aligned_face_image = aligned_face; + } + else + { + cv::vconcat(this->aligned_face_image, aligned_face, this->aligned_face_image); + } } void Visualizer::SetObservationHOG(const cv::Mat_& hog_descriptor, int num_cols, int num_rows) { if(vis_hog) { - Visualise_FHOG(hog_descriptor, num_rows, num_cols, this->hog_image); + if (this->hog_image.empty()) + { + Visualise_FHOG(hog_descriptor, num_rows, num_cols, this->hog_image); + } + else + { + cv::Mat tmp_hog; + Visualise_FHOG(hog_descriptor, num_rows, num_cols, tmp_hog); + cv::vconcat(this->hog_image, tmp_hog, this->hog_image); + } } + }