Merge branch 'develop'

.gitignore

@@ -28,3 +28,9 @@ exe/FeatureExtraction/out_bp4d/
 matlab_runners/Action Unit Experiments/out_bp4d/
 matlab_runners/Action Unit Experiments/out_SEMAINE/
 x64/Debug/
+matlab_runners/Action Unit Experiments/out_unbc/
+matlab_runners/Action Unit Experiments/out_bosph/
+matlab_runners/Action Unit Experiments/out_DISFA/
+matlab_runners/Action Unit Experiments/out_fera/
+matlab_runners/Demos/output_features_seq/
+matlab_runners/Demos/output_features_vid/

exe/FaceLandmarkImg/FaceLandmarkImg.cpp

@@ -188,26 +188,64 @@ void write_out_pose_landmarks(const string& outfeatures, const cv::Mat_<double>&
 	}
 }
 
-void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model)
+void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1, std::vector<std::pair<std::string, double>> au_intensities, std::vector<std::pair<std::string, double>> au_occurences)
 {
 	create_directory_from_file(outfeatures);
 	std::ofstream featuresFile;
 	featuresFile.open(outfeatures);
 
-	if(featuresFile.is_open())
+	if (featuresFile.is_open())
 	{
 		int n = clnf_model.patch_experts.visibilities[0][0].rows;
 		featuresFile << "version: 1" << endl;
 		featuresFile << "npoints: " << n << endl;
 		featuresFile << "{" << endl;
 
-		for (int i = 0; i < n; ++ i)
+		for (int i = 0; i < n; ++i)
 		{
 			// Use matlab format, so + 1
-			featuresFile << clnf_model.detected_landmarks.at<double>(i) + 1 << " " << clnf_model.detected_landmarks.at<double>(i+n) + 1 << endl;
+			featuresFile << clnf_model.detected_landmarks.at<double>(i) + 1 << " " << clnf_model.detected_landmarks.at<double>(i + n) + 1 << endl;
 		}
 		featuresFile << "}" << endl;
 
+		// Do the pose and eye gaze if present as well
+		featuresFile << "pose: eul_x, eul_y, eul_z: " << endl;
+		featuresFile << "{" << endl;
+		featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl;
+		featuresFile << "}" << endl;
+
+		// Do the pose and eye gaze if present as well
+		featuresFile << "gaze: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl;
+		featuresFile << "{" << endl;
+		featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl;
+		featuresFile << "}" << endl;
+
+		// Do the au intensities
+		featuresFile << "au intensities: " << au_intensities.size() << endl;
+		featuresFile << "{" << endl;
+
+		for (int i = 0; i < au_intensities.size(); ++i)
+		{
+			// Use matlab format, so + 1
+			featuresFile << au_intensities[i].first << " " << au_intensities[i].second << endl;
+		}
+
+		featuresFile << "}" << endl;
+
+		// Do the au occurences
+		featuresFile << "au occurences: " << au_occurences.size() << endl;
+		featuresFile << "{" << endl;
+
+		for (int i = 0; i < au_occurences.size(); ++i)
+		{
+			// Use matlab format, so + 1
+			featuresFile << au_occurences[i].first << " " << au_occurences[i].second << endl;
+		}
+
+		featuresFile << "}" << endl;
+
+
+		featuresFile.close();
 	}
 }
 
@@ -315,6 +353,45 @@ int main (int argc, char **argv)
 	cv::CascadeClassifier classifier(det_parameters.face_detector_location);
 	dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector();
 
+	// Loading the AU prediction models
+	string au_loc = "AU_predictors/AU_all_static.txt";
+
+	if (!boost::filesystem::exists(boost::filesystem::path(au_loc)))
+	{
+		boost::filesystem::path loc = boost::filesystem::path(arguments[0]).parent_path() / au_loc;
+
+		if (boost::filesystem::exists(loc))
+		{
+			au_loc = loc.string();
+		}
+		else
+		{
+			cout << "Can't find AU prediction files, exiting" << endl;
+			return 0;
+		}
+	}
+
+	// Used for image masking for AUs
+	string tri_loc;
+	if (boost::filesystem::exists(boost::filesystem::path("model/tris_68_full.txt")))
+	{
+		std::ifstream triangulation_file("model/tris_68_full.txt");
+		tri_loc = "model/tris_68_full.txt";
+	}
+	else
+	{
+		boost::filesystem::path loc = boost::filesystem::path(arguments[0]).parent_path() / "model/tris_68_full.txt";
+		tri_loc = loc.string();
+
+		if (!exists(loc))
+		{
+			cout << "Can't find triangulation files, exiting" << endl;
+			return 0;
+		}
+	}
+
+	FaceAnalysis::FaceAnalyser face_analyser(vector<cv::Vec3d>(), 0.7, 112, 112, au_loc, tri_loc);
+
 	bool visualise = !det_parameters.quiet_mode;
 
 	// Do some image loading
@@ -396,6 +473,8 @@ int main (int argc, char **argv)
 
 				}
 
+				auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false);
+
 				// Writing out the detected landmarks (in an OS independent manner)
 				if(!output_landmark_locations.empty())
 				{
@@ -412,7 +491,7 @@ int main (int argc, char **argv)
 					boost::filesystem::path fname = out_feat_path.filename().replace_extension("");
 					boost::filesystem::path ext = out_feat_path.extension();
 					string outfeatures = dir.string() + preferredSlash + fname.string() + string(name) + ext.string();
-					write_out_landmarks(outfeatures, clnf_model);
+					write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second);
 				}
 
 				if (!output_pose_locations.empty())
@@ -503,11 +582,13 @@ int main (int argc, char **argv)
 				FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false);
 			}
 
+			auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false);
+
 			// Writing out the detected landmarks
 			if(!output_landmark_locations.empty())
 			{
 				string outfeatures = output_landmark_locations.at(i);
-				write_out_landmarks(outfeatures, clnf_model);
+				write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second);
 			}
 
 			// Writing out the detected landmarks

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -153,7 +153,7 @@ void create_directory(string output_path)
 }
 
 void get_output_feature_params(vector<string> &output_similarity_aligned, vector<string> &output_hog_aligned_files, double &similarity_scale,
-	int &similarity_size, bool &grayscale, bool &rigid, bool& verbose,
+	int &similarity_size, bool &grayscale, bool &rigid, bool& verbose, bool& dynamic,
 	bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze,
 	vector<string> &arguments);
 
@@ -235,7 +235,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
 	cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
 	const FaceAnalysis::FaceAnalyser& face_analyser);
 
-void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file);
+void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic);
 
 
 int main (int argc, char **argv)
@@ -306,6 +306,7 @@ int main (int argc, char **argv)
 	bool grayscale = false;	
 	bool video_output = false;
 	bool rigid = false;	
+	bool dynamic = true; // Indicates if a dynamic AU model should be used (dynamic is useful if the video is long enough to include neutral expressions)
 	int num_hog_rows;
 	int num_hog_cols;
 
@@ -318,17 +319,14 @@ int main (int argc, char **argv)
 	bool output_AUs = true;
 	bool output_gaze = true;
 
-	get_output_feature_params(output_similarity_align, output_hog_align_files, sim_scale, sim_size, grayscale, rigid, verbose, 
+	get_output_feature_params(output_similarity_align, output_hog_align_files, sim_scale, sim_size, grayscale, rigid, verbose, dynamic,
 		output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, arguments);
 	
 	// Used for image masking
 
-	cv::Mat_<int> triangulation;
 	string tri_loc;
 	if(boost::filesystem::exists(path("model/tris_68_full.txt")))
 	{
-		std::ifstream triangulation_file("model/tris_68_full.txt");
-		LandmarkDetector::ReadMat(triangulation_file, triangulation);
 		tri_loc = "model/tris_68_full.txt";
 	}
 	else
@@ -336,12 +334,7 @@ int main (int argc, char **argv)
 		path loc = path(arguments[0]).parent_path() / "model/tris_68_full.txt";
 		tri_loc = loc.string();
 
-		if(exists(loc))
+		if(!exists(loc))
-		{
-			std::ifstream triangulation_file(loc.string());
-			LandmarkDetector::ReadMat(triangulation_file, triangulation);
-		}
-		else
 		{
 			cout << "Can't find triangulation files, exiting" << endl;
 			return 0;
@@ -359,13 +352,24 @@ int main (int argc, char **argv)
 	int curr_img = -1;
 
 	string au_loc;
-	if(boost::filesystem::exists(path("AU_predictors/AU_all_best.txt")))
+
+	string au_loc_local;
+	if (dynamic)
 	{
-		au_loc = "AU_predictors/AU_all_best.txt";
+		au_loc_local = "AU_predictors/AU_all_best.txt";
 	}
 	else
 	{
-		path loc = path(arguments[0]).parent_path() / "AU_predictors/AU_all_best.txt";
+		au_loc_local = "AU_predictors/AU_all_static.txt";
+	}
+
+	if(boost::filesystem::exists(path(au_loc_local)))
+	{
+		au_loc = au_loc_local;
+	}
+	else
+	{
+		path loc = path(arguments[0]).parent_path() / au_loc_local;
 
 		if(exists(loc))
 		{
@@ -684,16 +688,10 @@ int main (int argc, char **argv)
 		
 		output_file.close();
 
-		if(output_files.size() > 0)
+		if(output_files.size() > 0 && output_AUs)
-		{
-			
-			// If the video is long enough post-process it for AUs
-			if (output_AUs && frame_count > 100)
 		{
 			cout << "Postprocessing the Action Unit predictions" << endl;
-
+			post_process_output_file(face_analyser, output_files[f_n], dynamic);
-				post_process_output_file(face_analyser, output_files[f_n]);
-			}
 		}
 		// Reset the models for the next video
 		face_analyser.Reset();
@@ -718,7 +716,7 @@ int main (int argc, char **argv)
 }
 
 // Allows for post processing of the AU signal
-void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file)
+void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic)
 {
 
 	vector<double> certainties;
@@ -728,8 +726,8 @@ void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string
 	vector<std::pair<std::string, vector<double>>> predictions_class;
 
 	// Construct the new values to overwrite the output file with
-	face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps);
+	face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic);
-	face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps);
+	face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic);
 
 	int num_class = predictions_class.size();
 	int num_reg = predictions_reg.size();
@@ -1024,7 +1022,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
 
 
 void get_output_feature_params(vector<string> &output_similarity_aligned, vector<string> &output_hog_aligned_files, double &similarity_scale,
-	int &similarity_size, bool &grayscale, bool &rigid, bool& verbose,
+	int &similarity_size, bool &grayscale, bool &rigid, bool& verbose, bool& dynamic,
 	bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze,
 	vector<string> &arguments)
 {
@@ -1041,6 +1039,9 @@ void get_output_feature_params(vector<string> &output_similarity_aligned, vector
 	string input_root = "";
 	string output_root = "";
 
+	// By default the model is dynamic
+	dynamic = true;
+
 	// First check if there is a root argument (so that videos and outputs could be defined more easilly)
 	for (size_t i = 0; i < arguments.size(); ++i)
 	{
@@ -1088,6 +1089,10 @@ void get_output_feature_params(vector<string> &output_similarity_aligned, vector
 		{
 			rigid = true;
 		}
+		else if (arguments[i].compare("-au_static") == 0)
+		{
+			dynamic = false;
+		}
 		else if (arguments[i].compare("-g") == 0)
 		{
 			grayscale = true;

lib/local/FaceAnalyser/AU_predictors/AU_all_best.txt

@@ -1,20 +1,35 @@
-svr_disfa/AU_1_static.dat AU01
+svm_combined/AU_1_dynamic.dat AU01
-svr_disfa/AU_2_dyn.dat AU02
+svm_combined/AU_2_dynamic.dat AU02
-svr_disfa/AU_4_static.dat AU04
+svm_combined/AU_4_static.dat AU04
-svm_combined/AU_4_dynamic_combined_all.dat AU04
+svm_combined/AU_5_static.dat AU05
-svr_disfa/AU_5_dyn.dat AU05
+svm_combined/AU_6_static.dat AU06
-svr_combined/AU_6_static_intensity_combined.dat AU06
+svm_combined/AU_7_static.dat AU07
-svr_disfa/AU_9_dyn.dat AU09
+svm_combined/AU_9_dynamic.dat AU09
-svr_bp4d/AU_10_static_intensity.dat AU10
+svm_combined/AU_10_static.dat AU10
-svm_combined/AU_12_static_combined_all.dat AU12
+svm_combined/AU_12_static.dat AU12
-svr_combined/AU_12_static_intensity_combined.dat AU12
+svm_combined/AU_14_static.dat AU14
-svr_bp4d/AU_14_static_intensity.dat AU14
+svm_combined/AU_15_dynamic.dat AU15
-svr_disfa/AU_15_dyn.dat AU15
+svm_combined/AU_17_dynamic.dat AU17
-svm_combined/AU_15_dynamic_combined_all.dat AU15
+svm_combined/AU_20_dynamic.dat AU20
-svr_combined/AU_17_static_intensity_combined.dat AU17
+svm_combined/AU_23_static.dat AU23
-svr_disfa/AU_20_dyn.dat AU20
+svm_combined/AU_25_dynamic.dat AU25
-svm_bp4d/AU_23_static.dat AU23
+svm_combined/AU_26_dynamic.dat AU26
-svr_disfa/AU_25_static.dat AU25
+svm_combined/AU_28_static.dat AU28
-svr_disfa/AU_26_dyn.dat AU26
+svm_combined/AU_45_dynamic.dat AU45
-svm_semaine/AU_28_static.dat AU28
+svr_combined/AU_1_dynamic_intensity_comb.dat AU01
-svm_semaine/AU_45_dynamic.dat AU45
+svr_combined/AU_2_dynamic_intensity_comb.dat AU02
+svr_combined/AU_4_static_intensity_comb.dat AU04
+svr_combined/AU_5_dynamic_intensity.dat AU05
+svr_combined/AU_6_static_intensity_comb.dat AU06
+svr_combined/AU_7_static_intensity_comb.dat AU07
+svr_combined/AU_9_dynamic_intensity.dat AU09
+svr_combined/AU_10_static_intensity_comb.dat AU10
+svr_combined/AU_12_static_intensity_comb.dat AU12
+svr_combined/AU_14_static_intensity.dat AU14
+svr_combined/AU_15_dynamic_intensity_comb.dat AU15
+svr_combined/AU_17_dynamic_intensity_comb.dat AU17
+svr_combined/AU_20_dynamic_intensity.dat AU20
+svr_combined/AU_23_dynamic_intensity_comb.dat AU23
+svr_combined/AU_25_dynamic_intensity_comb.dat AU25
+svr_combined/AU_26_dynamic_intensity_comb.dat AU26
+svr_combined/AU_45_dynamic_intensity_comb.dat AU45

lib/local/FaceAnalyser/AU_predictors/AU_all_static.txt

@@ -1,23 +1,35 @@
-svr_disfa/AU_1_static.dat AU01
+svm_combined/AU_1_static.dat AU01
-svr_disfa/AU_2_static.dat AU02
+svm_combined/AU_2_static.dat AU02
-svr_disfa/AU_4_static.dat AU04
+svm_combined/AU_4_static.dat AU04
-svm_combined/AU_4_static_combined_all.dat AU04
+svm_combined/AU_5_static.dat AU05
-svr_disfa/AU_5_static.dat AU05
+svm_combined/AU_6_static.dat AU06
-svr_combined/AU_6_static_intensity_combined.dat AU06
+svm_combined/AU_7_static.dat AU07
-svm_bp4d/AU_7_static.dat AU07
+svm_combined/AU_9_static.dat AU09
-svr_disfa/AU_9_static.dat AU09
+svm_combined/AU_10_static.dat AU10
-svr_bp4d/AU_10_static_intensity.dat AU10
+svm_combined/AU_12_static.dat AU12
-svm_combined/AU_12_static_combined_all.dat AU12
+svm_combined/AU_14_static.dat AU14
-svr_combined/AU_12_static_intensity_combined.dat AU12
+svm_combined/AU_15_static.dat AU15
-svr_bp4d/AU_14_static_intensity.dat AU14
+svm_combined/AU_17_static.dat AU17
-svr_disfa/AU_15_static.dat AU15
+svm_combined/AU_20_static.dat AU20
-svm_combined/AU_15_static_combined_all.dat AU15
+svm_combined/AU_23_static.dat AU23
-svr_combined/AU_17_static_intensity_combined.dat AU17
+svm_combined/AU_25_static.dat AU25
-svm_combined/AU_17_static_combined_all.dat AU17
+svm_combined/AU_26_static.dat AU26
-svr_disfa/AU_20_static.dat AU20
+svm_combined/AU_28_static.dat AU28
-svm_bp4d/AU_23_static.dat AU23
+svm_combined/AU_45_static.dat AU45
-svr_disfa/AU_25_static.dat AU25
+svr_combined/AU_1_static_intensity.dat AU01
-svm_combined/AU_25_static_combined_all.dat AU25
+svr_combined/AU_2_static_intensity_comb.dat AU02
-svr_disfa/AU_26_static.dat AU26
+svr_combined/AU_4_static_intensity_comb.dat AU04
-svm_semaine/AU_28_static.dat AU28
+svr_combined/AU_5_static_intensity.dat AU05
-svm_semaine/AU_45_static.dat AU45
+svr_combined/AU_6_static_intensity_comb.dat AU06
+svr_combined/AU_7_static_intensity_comb.dat AU07
+svr_combined/AU_9_static_intensity.dat AU09
+svr_combined/AU_10_static_intensity_comb.dat AU10
+svr_combined/AU_12_static_intensity_comb.dat AU12
+svr_combined/AU_14_static_intensity.dat AU14
+svr_combined/AU_15_static_intensity_comb.dat AU15
+svr_combined/AU_17_static_intensity_comb.dat AU17
+svr_combined/AU_20_static_intensity.dat AU20
+svr_combined/AU_23_static_intensity_comb.dat AU23
+svr_combined/AU_25_static_intensity.dat AU25
+svr_combined/AU_26_static_intensity_comb.dat AU26
+svr_combined/AU_45_static_intensity_comb.dat AU45