From a916e68fdb25e3ab8f46b40ce78740067aa6223b Mon Sep 17 00:00:00 2001
From: Tadas Baltrusaitis <tadyla@gmail.com>
Date: Thu, 18 Jan 2018 15:47:58 +0000
Subject: [PATCH] Gaze and AU support when recording

---
 gui/OpenFaceOffline/MainWindow.xaml.cs |  4 ++-
 lib/local/CppInerop/RecorderInterop.h  | 35 ++++++++++++++++++++------
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs
index b905fea..5f4856e 100644
--- a/gui/OpenFaceOffline/MainWindow.xaml.cs
+++ b/gui/OpenFaceOffline/MainWindow.xaml.cs
@@ -451,7 +451,9 @@ namespace OpenFaceOffline
             var landmarks_3d_eyes = clnf_model.CalculateAllEyeLandmarks3D(fx, fy, cx, cy);
             recorder.SetObservationGaze(gaze.Item1, gaze.Item2, gaze_angle, landmarks_2d_eyes, landmarks_3d_eyes);
 
-            //open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
+            var au_regs = face_analyser.GetCurrentAUsReg();
+            var au_classes = face_analyser.GetCurrentAUsClass();
+            recorder.SetObservationActionUnits(au_regs, au_classes);
 
             //open_face_rec.SetObservationFaceAlign(sim_warped_img);
             //open_face_rec.WriteObservation();
diff --git a/lib/local/CppInerop/RecorderInterop.h b/lib/local/CppInerop/RecorderInterop.h
index 2bd2a67..2b404f0 100644
--- a/lib/local/CppInerop/RecorderInterop.h
+++ b/lib/local/CppInerop/RecorderInterop.h
@@ -116,23 +116,20 @@ namespace UtilitiesOF {
 			cv::Vec2d gaze_angle_cv(gaze_angle->Item1, gaze_angle->Item2);
 
 			// Construct an OpenCV matrix from the landmarks
-			cv::Mat_<double> landmarks_2D_mat(landmarks_2D->Count * 2, 1, 0.0);
+			std::vector<cv::Point2d> landmarks_2D_cv;
 			for (int i = 0; i < landmarks_2D->Count; ++i)
 			{
-				landmarks_2D_mat.at<double>(i, 0) = landmarks_2D[i]->Item1;
-				landmarks_2D_mat.at<double>(i + landmarks_2D->Count, 0) = landmarks_2D[i]->Item2;
+				landmarks_2D_cv.push_back(cv::Point2d(landmarks_2D[i]->Item1, landmarks_2D[i]->Item2));
 			}
 
 			// Construct an OpenCV matrix from the landmarks
-			cv::Mat_<double> landmarks_3D_mat(landmarks_3D->Count * 3, 1, 0.0);
+			std::vector<cv::Point3d> landmarks_3D_cv;
 			for (int i = 0; i < landmarks_3D->Count; ++i)
 			{
-				landmarks_3D_mat.at<double>(i, 0) = landmarks_3D[i]->Item1;
-				landmarks_3D_mat.at<double>(i + landmarks_3D->Count, 0) = landmarks_3D[i]->Item2;
-				landmarks_3D_mat.at<double>(i + 2 * landmarks_3D->Count, 0) = landmarks_3D[i]->Item3;
+				landmarks_3D_cv.push_back(cv::Point3d(landmarks_3D[i]->Item1, landmarks_3D[i]->Item2, landmarks_3D[i]->Item3));
 			}
 
-			m_recorder->SetObservationGaze(gaze_direction0_cv, gaze_direction1_cv, gaze_angle_cv, landmarks_2D_mat, landmarks_3D_mat);
+			m_recorder->SetObservationGaze(gaze_direction0_cv, gaze_direction1_cv, gaze_angle_cv, landmarks_2D_cv, landmarks_3D_cv);
 		}
 
 		// Setting the observations
@@ -142,6 +139,28 @@ namespace UtilitiesOF {
 			m_recorder->SetObservationPose(pose_vec);
 		}
 
+		void SetObservationActionUnits(Dictionary<System::String^, double>^ au_regs, Dictionary<System::String^, double>^ au_class)
+		{
+			std::vector<std::pair<std::string, double> > au_regs_std;
+			auto enum_reg = au_regs->GetEnumerator();
+			while (enum_reg.MoveNext())
+			{
+				std::string au_name = msclr::interop::marshal_as<std::string>(enum_reg.Current.Key);
+				double value = (double)enum_reg.Current.Value;
+				au_regs_std.push_back(std::pair<std::string, double>(au_name, value));
+			}
+
+			std::vector<std::pair<std::string, double> > au_class_std;
+			auto enum_class = au_class->GetEnumerator();
+			while (enum_class.MoveNext())
+			{
+				std::string au_name = msclr::interop::marshal_as<std::string>(enum_class.Current.Key);
+				double value = (double)enum_class.Current.Value;
+				au_class_std.push_back(std::pair<std::string, double>(au_name, value));
+			} 
+			m_recorder->SetObservationActionUnits(au_regs_std, au_class_std);
+		}
+
 		void SetObservationLandmarks(List<System::Tuple<double, double>^>^ landmarks_2D, List<System::Tuple<double, double, double>^>^ landmarks_3D, List<double>^ params_global, List<double>^ params_local, double confidence, bool success)
 		{
 			// Construct an OpenCV matrix from the landmarks