- Fixing Windows debug build due to linking to OpenCV.

- Adding a -nomask flag for extracting aligned faces

lib/3rdParty/OpenCV3.4/openCV3.4.props

@@ -8,8 +8,8 @@
       <AdditionalIncludeDirectories>$(SolutionDir)lib\3rdParty\OpenCV3.4\include\opencv;$(SolutionDir)lib\3rdParty\OpenCV3.4\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <AdditionalLibraryDirectories>$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>opencv_world340.lib;opencv_world340d.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>opencv_world340.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <PostBuildEvent>
       <Command>xcopy /I /E /Y /D /C "$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\bin\$(Configuration)" "$(OutDir)"

lib/local/FaceAnalyser/include/FaceAnalyser.h

@@ -226,6 +226,7 @@ private:
 	int align_width_au;
 	int align_height_au;
 
+	bool align_mask;
 	double align_scale_out;
 	int align_width_out;
 	int align_height_out;

lib/local/FaceAnalyser/include/FaceAnalyserParameters.h

@@ -64,11 +64,12 @@ public:
 	bool grayscale;
 
 	// Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense
-	void setAlignedOutput(int output_size, double scale=-1);
+	void setAlignedOutput(int output_size, double scale=-1, bool masked = true);
 	// This will also change the model location
 	void OptimizeForVideos();
 	void OptimizeForImages();
 
+	double getAlignMask() const { return sim_align_face_mask; }
 	double getSimScaleOut() const { return sim_scale_out; }
 	int getSimSizeOut() const { return sim_size_out; }
 	bool getDynamic() const { return dynamic; }
@@ -83,6 +84,9 @@ private:
 	double sim_scale_out;
 	int sim_size_out;
 
+	// Should aligned face be masked out from background
+	bool sim_align_face_mask;
+
 	// Should a video stream be assumed
 	bool dynamic;
 

lib/local/FaceAnalyser/src/FaceAnalyser.cpp

@@ -63,6 +63,7 @@ FaceAnalyser::FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_anal
 {
 	this->Read(face_analyser_params.getModelLoc());
 		
+	align_mask = face_analyser_params.getAlignMask();
 	align_scale_out = face_analyser_params.getSimScaleOut();
 	align_width_out = face_analyser_params.getSimSizeOut();
 	align_height_out = face_analyser_params.getSimSizeOut();
@@ -256,17 +257,24 @@ void FaceAnalyser::PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, cons
 	cv::Mat_<float> params_local;
 	pdm.CalcParams(params_global, params_local, detected_landmarks);
 
-	// First align the face
+	// The aligned face requirement for AUs
-	AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, 0.7, 112, 112);
+	AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au);
-	
+
-	// If the output requirement matches use the already computed one, else compute it again
+	// If the aligned face for AU matches the output requested one, just reuse it, else compute it
-	if (align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au)
+	if (align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au && align_mask)
 	{
 		aligned_face_for_output = aligned_face_for_au.clone();
 	}
 	else
 	{
-		AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out);
+		if (align_mask)
+		{
+			AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out);
+		}
+		else
+		{
+			AlignFace(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, true, align_scale_out, align_width_out, align_height_out);
+		}
 	}
 
 	// Extract HOG descriptor from the frame and convert it to a useable format
@@ -315,6 +323,7 @@ void FaceAnalyser::PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, cons
 
 }
 
+
 void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_<float>& detected_landmarks, bool success, double timestamp_seconds, bool online)
 {
 
@@ -333,14 +342,21 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_<float>& det
 		// The aligned face requirement for AUs
 		AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au);
 
-		// If the output requirement matches use the already computed one, else compute it again
+		// If the aligned face for AU matches the output requested one, just reuse it, else compute it
-		if(align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au)
+		if (align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au && align_mask)
 		{
 			aligned_face_for_output = aligned_face_for_au.clone();
 		}
 		else
 		{
-			AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out);
+			if (align_mask)
+			{
+				AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out);
+			}
+			else
+			{
+				AlignFace(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, true, align_scale_out, align_width_out, align_height_out);
+			}
 		}
 	}
 	else

lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp

@@ -88,6 +88,11 @@ FaceAnalyserParameters::FaceAnalyserParameters(vector<string> &arguments):root()
 			grayscale = true;
 			valid[i] = false;
 		}
+		else if (arguments[i].compare("-nomask") == 0)
+		{
+			sim_align_face_mask = false;
+			valid[i] = false;
+		}
 		else if (arguments[i].compare("-simscale") == 0)
 		{
 			sim_scale_out = stod(arguments[i + 1]);
@@ -155,6 +160,7 @@ void FaceAnalyserParameters::init()
 	this->grayscale = false;
 	this->sim_scale_out = 0.7;
 	this->sim_size_out = 112;
+	this->sim_align_face_mask = true;
 
 	this->model_location = "AU_predictors/main_dynamic_svms.txt";
 
@@ -184,13 +190,15 @@ void FaceAnalyserParameters::init()
 }
 
 // Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense
-void FaceAnalyserParameters::setAlignedOutput(int output_size, double scale)
+void FaceAnalyserParameters::setAlignedOutput(int output_size, double scale, bool masked)
 {
 	this->sim_size_out = output_size;
 	// If we set the size but not the scale, adapt the scale to the right size
 	if (scale ==-1) this->sim_scale_out = sim_size_out * (0.7 / 112.0);
 	else this->sim_scale_out = sim_scale_out;
 
+	this->sim_align_face_mask = masked;
+
 }
 // This will also change the model location
 void FaceAnalyserParameters::OptimizeForVideos()

lib/local/FaceAnalyser/src/Face_utils.cpp

@@ -114,7 +114,7 @@ namespace FaceAnalysis
 	}
 
 	// Aligning a face to a common reference frame
-	void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_<float>& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid, float sim_scale, int out_width, int out_height)
+	void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_<float>& detected_landmarks, cv::Vec6f params_global, const PDM& pdm, bool rigid, float sim_scale, int out_width, int out_height)
 	{
 		// Will warp to scaled mean shape
 		cv::Mat_<float> similarity_normalised_shape = pdm.mean_shape * sim_scale;
@@ -131,19 +131,18 @@ namespace FaceAnalysis
 			extract_rigid_points(source_landmarks, destination_landmarks);
 		}
 
-		// TODO rem the doubles here
+		cv::Matx22f scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks);
-		cv::Matx22d scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks);
+		cv::Matx23f warp_matrix;
-		cv::Matx23d warp_matrix;
 
 		warp_matrix(0,0) = scale_rot_matrix(0,0);
 		warp_matrix(0,1) = scale_rot_matrix(0,1);
 		warp_matrix(1,0) = scale_rot_matrix(1,0);
 		warp_matrix(1,1) = scale_rot_matrix(1,1);
 
-		double tx = params_global[4];
+		float tx = params_global[4];
-		double ty = params_global[5];
+		float ty = params_global[5];
 
-		cv::Vec2d T(tx, ty);
+		cv::Vec2f T(tx, ty);
 		T = scale_rot_matrix * T;
 
 		// Make sure centering is correct