From 0ac039d7083c6f955163c5d310ab87c216cecad7 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 30 Oct 2017 09:43:03 +0000 Subject: [PATCH] Preparing for processing of image files, and a new image output format. --- gui/OpenFaceOffline/MainWindow.xaml.cs | 23 ++++++---- gui/OpenFaceOffline/Recorder.cs | 6 +-- lib/local/CppInerop/FaceAnalyserInterop.h | 45 ++++++++++++------- lib/local/CppInerop/LandmarkDetectorInterop.h | 44 +++++++++--------- 4 files changed, 67 insertions(+), 51 deletions(-) diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index d8ac6fb..d6075ae 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -335,7 +335,20 @@ namespace OpenFaceOffline return; } - List>> landmark_detections = ProcessImage(clnf_model, face_model_params, frame, grayFrame); + var landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayFrame, face_model_params); + + // Go over all detected faces + for(int i = 0; i < landmark_detections.Count; ++i) + { + + // Predict action units + var au_preds = face_analyser.PredictStaticAUs(grayFrame, landmark_detections[i]); + + // Record the predictions + //String output_name = record_root + + //Recorder.RecordImg(); + + } List landmark_points = new List(); @@ -731,14 +744,6 @@ namespace OpenFaceOffline } - private List>> ProcessImage(CLNF clnf_model, FaceModelParameters clnf_params, RawImage frame, RawImage grayscale_frame) - { - List>> landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayscale_frame, clnf_params); - return landmark_detections; - - } - - // ---------------------------------------------------------- // Mode handling (image, video) // ---------------------------------------------------------- diff --git a/gui/OpenFaceOffline/Recorder.cs b/gui/OpenFaceOffline/Recorder.cs index 71b16ff..2248bf6 100644 --- a/gui/OpenFaceOffline/Recorder.cs +++ b/gui/OpenFaceOffline/Recorder.cs @@ -296,11 +296,9 @@ namespace OpenFaceOffline face_analyser.PostProcessOutputFile(out_filename); } - static void RecordImg(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, - bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG, - CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model) + static void RecordImg(string out_root, string filename, CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy) { - + // Points, pose, gaze, aus } } diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index c996786..de86d2f 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -70,6 +70,8 @@ #undef generic #endif +using namespace System::Collections::Generic; + #pragma managed namespace FaceAnalyser_Interop { @@ -185,7 +187,7 @@ public: face_analyser->PostprocessOutputFile(msclr::interop::marshal_as(file)); } - void AddNextFrame(OpenCVWrappers::RawImage^ frame, System::Collections::Generic::List^>^ landmarks, bool success, bool online, bool vis_hog) { + void AddNextFrame(OpenCVWrappers::RawImage^ frame, List^>^ landmarks, bool success, bool online, bool vis_hog) { // Construct an OpenCV matric from the landmarks cv::Mat_ landmarks_mat(landmarks->Count * 2, 1, 0.0); @@ -211,7 +213,9 @@ public: } // Predicting AUs from a single image - System::Collections::Generic::Dictionary^ PredictStaticAUs(OpenCVWrappers::RawImage^ frame, System::Collections::Generic::List^>^ landmarks, bool success, bool vis_hog) { + System::Tuple^, Dictionary^>^ + PredictStaticAUs(OpenCVWrappers::RawImage^ frame, List^>^ landmarks) + { // Construct an OpenCV matric from the landmarks cv::Mat_ landmarks_mat(landmarks->Count * 2, 1, 0.0); @@ -221,26 +225,33 @@ public: landmarks_mat.at(i + landmarks->Count, 0) = landmarks[i]->Item2; } - face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, false, vis_hog); + auto aus = face_analyser->PredictStaticAUs(frame->Mat, landmarks_mat, false); - face_analyser->GetLatestHOG(*hog_features, *num_rows, *num_cols); + auto AU_predictions_intensity = aus.first; + auto AU_predictions_occurence = aus.second; - face_analyser->GetLatestAlignedFace(*aligned_face); + auto au_intensities = gcnew Dictionary(); + auto au_occurences = gcnew Dictionary(); - *good_frame = success; - - if (vis_hog) + for (auto p : AU_predictions_intensity) { - *visualisation = face_analyser->GetLatestHOGDescriptorVisualisation(); + au_intensities->Add(gcnew System::String(p.first.c_str()), p.second); } + for (auto p : AU_predictions_occurence) + { + au_occurences->Add(gcnew System::String(p.first.c_str()), p.second); + } + + return gcnew System::Tuple^, Dictionary^>(au_intensities, au_occurences); + } - System::Collections::Generic::List^ GetClassActionUnitsNames() + List^ GetClassActionUnitsNames() { auto names = face_analyser->GetAUClassNames(); - auto names_ret = gcnew System::Collections::Generic::List(); + auto names_ret = gcnew List(); for(std::string name : names) { @@ -251,11 +262,11 @@ public: } - System::Collections::Generic::List^ GetRegActionUnitsNames() + List^ GetRegActionUnitsNames() { auto names = face_analyser->GetAURegNames(); - auto names_ret = gcnew System::Collections::Generic::List(); + auto names_ret = gcnew List(); for(std::string name : names) { @@ -266,10 +277,10 @@ public: } - System::Collections::Generic::Dictionary^ GetCurrentAUsClass() + Dictionary^ GetCurrentAUsClass() { auto classes = face_analyser->GetCurrentAUsClass(); - auto au_classes = gcnew System::Collections::Generic::Dictionary(); + auto au_classes = gcnew Dictionary(); for(auto p: classes) { @@ -278,10 +289,10 @@ public: return au_classes; } - System::Collections::Generic::Dictionary^ GetCurrentAUsReg() + Dictionary^ GetCurrentAUsReg() { auto preds = face_analyser->GetCurrentAUsReg(); - auto au_preds = gcnew System::Collections::Generic::Dictionary(); + auto au_preds = gcnew Dictionary(); for(auto p: preds) { diff --git a/lib/local/CppInerop/LandmarkDetectorInterop.h b/lib/local/CppInerop/LandmarkDetectorInterop.h index 417b77e..af7e339 100644 --- a/lib/local/CppInerop/LandmarkDetectorInterop.h +++ b/lib/local/CppInerop/LandmarkDetectorInterop.h @@ -68,6 +68,8 @@ #undef generic #endif +using namespace System::Collections::Generic; + #pragma managed namespace CppInterop { @@ -202,9 +204,9 @@ namespace CppInterop { return ::LandmarkDetector::DetectLandmarksInImage(image->Mat, *clnf, *modelParams->getParams()); } - System::Collections::Generic::List^>^>^ DetectMultiFaceLandmarksInImage(OpenCVWrappers::RawImage^ image, FaceModelParameters^ modelParams) { + List^>^>^ DetectMultiFaceLandmarksInImage(OpenCVWrappers::RawImage^ image, FaceModelParameters^ modelParams) { - auto all_landmarks = gcnew System::Collections::Generic::List^>^>(); + auto all_landmarks = gcnew List^>^>(); // Detect faces in an image vector > face_detections; @@ -224,7 +226,7 @@ namespace CppInterop { // if there are multiple detections go through them bool success = ::LandmarkDetector::DetectLandmarksInImage(image->Mat, face_detections[face], *clnf, *modelParams->getParams()); - auto landmarks_curr = gcnew System::Collections::Generic::List^>(); + auto landmarks_curr = gcnew List^>(); if(clnf->detected_landmarks.cols == 1) { int n = clnf->detected_landmarks.rows / 2; @@ -248,7 +250,7 @@ namespace CppInterop { return all_landmarks; } - void GetPoseWRTCamera(System::Collections::Generic::List^ pose, double fx, double fy, double cx, double cy) { + void GetPoseWRTCamera(List^ pose, double fx, double fy, double cx, double cy) { auto pose_vec = ::LandmarkDetector::GetPoseWRTCamera(*clnf, fx, fy, cx, cy); pose->Clear(); for(int i = 0; i < 6; ++i) @@ -257,7 +259,7 @@ namespace CppInterop { } } - void GetPose(System::Collections::Generic::List^ pose, double fx, double fy, double cx, double cy) { + void GetPose(List^ pose, double fx, double fy, double cx, double cy) { auto pose_vec = ::LandmarkDetector::GetPose(*clnf, fx, fy, cx, cy); pose->Clear(); for(int i = 0; i < 6; ++i) @@ -266,7 +268,7 @@ namespace CppInterop { } } - System::Collections::Generic::List^>^ CalculateVisibleLandmarks() { + List^>^ CalculateVisibleLandmarks() { vector vecLandmarks = ::LandmarkDetector::CalculateVisibleLandmarks(*clnf); auto landmarks = gcnew System::Collections::Generic::List^>(); @@ -277,10 +279,10 @@ namespace CppInterop { return landmarks; } - System::Collections::Generic::List^>^ CalculateAllLandmarks() { + List^>^ CalculateAllLandmarks() { vector vecLandmarks = ::LandmarkDetector::CalculateAllLandmarks(*clnf); - auto landmarks = gcnew System::Collections::Generic::List^>(); + auto landmarks = gcnew List^>(); for (cv::Point2d p : vecLandmarks) { landmarks->Add(gcnew System::Tuple(p.x, p.y)); } @@ -288,7 +290,7 @@ namespace CppInterop { return landmarks; } - System::Collections::Generic::List^>^ CalculateAllEyeLandmarks() { + List^>^ CalculateAllEyeLandmarks() { vector vecLandmarks = ::LandmarkDetector::CalculateAllEyeLandmarks(*clnf); auto landmarks = gcnew System::Collections::Generic::List^>(); @@ -299,7 +301,7 @@ namespace CppInterop { return landmarks; } - System::Collections::Generic::List^>^ CalculateVisibleEyeLandmarks() { + List^>^ CalculateVisibleEyeLandmarks() { vector vecLandmarks = ::LandmarkDetector::CalculateVisibleEyeLandmarks(*clnf); auto landmarks = gcnew System::Collections::Generic::List^>(); @@ -310,11 +312,11 @@ namespace CppInterop { return landmarks; } - System::Collections::Generic::List^ Calculate3DLandmarks(double fx, double fy, double cx, double cy) { + List^ Calculate3DLandmarks(double fx, double fy, double cx, double cy) { cv::Mat_ shape3D = clnf->GetShape(fx, fy, cx, cy); - auto landmarks_3D = gcnew System::Collections::Generic::List(); + auto landmarks_3D = gcnew List(); for(int i = 0; i < shape3D.cols; ++i) { @@ -326,7 +328,7 @@ namespace CppInterop { // Static functions from the LandmarkDetector namespace. - void DrawLandmarks(OpenCVWrappers::RawImage^ img, System::Collections::Generic::List^ landmarks) { + void DrawLandmarks(OpenCVWrappers::RawImage^ img, List^ landmarks) { vector vecLandmarks; @@ -339,13 +341,13 @@ namespace CppInterop { } - System::Collections::Generic::List^>^ CalculateBox(float fx, float fy, float cx, float cy) { + List^>^ CalculateBox(float fx, float fy, float cx, float cy) { cv::Vec6d pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy); vector> vecLines = ::LandmarkDetector::CalculateBox(pose, fx, fy, cx, cy); - auto lines = gcnew System::Collections::Generic::List^>(); + auto lines = gcnew List^>(); for(pair line : vecLines) { lines->Add(gcnew System::Tuple(System::Windows::Point(line.first.x, line.first.y), System::Windows::Point(line.second.x, line.second.y))); @@ -354,7 +356,7 @@ namespace CppInterop { return lines; } - void DrawBox(System::Collections::Generic::List^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) { + void DrawBox(List^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) { cv::Scalar color = cv::Scalar(r,g,b,1); vector> vecLines; @@ -378,9 +380,9 @@ namespace CppInterop { } // Getting the non-rigid shape parameters describing the facial expression - System::Collections::Generic::List^ GetNonRigidParams() + List^ GetNonRigidParams() { - auto non_rigid_params = gcnew System::Collections::Generic::List(); + auto non_rigid_params = gcnew List(); for (int i = 0; i < clnf->params_local.rows; ++i) { @@ -391,9 +393,9 @@ namespace CppInterop { } // Getting the rigid shape parameters describing face scale rotation and translation (scale,rotx,roty,rotz,tx,ty) - System::Collections::Generic::List^ GetRigidParams() + List^ GetRigidParams() { - auto rigid_params = gcnew System::Collections::Generic::List(); + auto rigid_params = gcnew List(); for (size_t i = 0; i < 6; ++i) { @@ -403,7 +405,7 @@ namespace CppInterop { } // Rigid params followed by non-rigid ones - System::Collections::Generic::List^ GetParams() + List^ GetParams() { auto all_params = GetRigidParams(); all_params->AddRange(GetNonRigidParams());