More work on AU - WIP

.gitignore

@@ -27,3 +27,4 @@ matlab_runners/Action Unit Experiments/out_SEMAINE/
 exe/FeatureExtraction/out_bp4d/
 x64/Debug/
 matlab_runners/Action Unit Experiments/out_unbc/
+matlab_runners/Action Unit Experiments/out_bosph/

exe/FaceLandmarkImg/FaceLandmarkImg.cpp

@@ -188,26 +188,64 @@ void write_out_pose_landmarks(const string& outfeatures, const cv::Mat_<double>&
 	}
 }
 
-void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model)
+void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1, std::vector<std::pair<std::string, double>> au_intensities, std::vector<std::pair<std::string, double>> au_occurences)
 {
 	create_directory_from_file(outfeatures);
 	std::ofstream featuresFile;
-	featuresFile.open(outfeatures);		
+	featuresFile.open(outfeatures);
 
-	if(featuresFile.is_open())
-	{	
+	if (featuresFile.is_open())
+	{
 		int n = clnf_model.patch_experts.visibilities[0][0].rows;
 		featuresFile << "version: 1" << endl;
 		featuresFile << "npoints: " << n << endl;
 		featuresFile << "{" << endl;
-		
-		for (int i = 0; i < n; ++ i)
+
+		for (int i = 0; i < n; ++i)
 		{
 			// Use matlab format, so + 1
-			featuresFile << clnf_model.detected_landmarks.at<double>(i) + 1 << " " << clnf_model.detected_landmarks.at<double>(i+n) + 1 << endl;
+			featuresFile << clnf_model.detected_landmarks.at<double>(i) + 1 << " " << clnf_model.detected_landmarks.at<double>(i + n) + 1 << endl;
 		}
-		featuresFile << "}" << endl;		
+		featuresFile << "}" << endl;
 
+		// Do the pose and eye gaze if present as well
+		featuresFile << "pose: eul_x, eul_y, eul_z: " << endl;
+		featuresFile << "{" << endl;
+		featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl;
+		featuresFile << "}" << endl;
+
+		// Do the pose and eye gaze if present as well
+		featuresFile << "gaze: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl;
+		featuresFile << "{" << endl;
+		featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl;
+		featuresFile << "}" << endl;
+
+		// Do the au intensities
+		featuresFile << "au intensities: " << au_intensities.size() << endl;
+		featuresFile << "{" << endl;
+
+		for (int i = 0; i < au_intensities.size(); ++i)
+		{
+			// Use matlab format, so + 1
+			featuresFile << au_intensities[i].first << " " << au_intensities[i].second << endl;
+		}
+
+		featuresFile << "}" << endl;
+
+		// Do the au occurences
+		featuresFile << "au occurences: " << au_occurences.size() << endl;
+		featuresFile << "{" << endl;
+
+		for (int i = 0; i < au_occurences.size(); ++i)
+		{
+			// Use matlab format, so + 1
+			featuresFile << au_occurences[i].first << " " << au_occurences[i].second << endl;
+		}
+
+		featuresFile << "}" << endl;
+
+
+		featuresFile.close();
 	}
 }
 
@@ -315,6 +353,45 @@ int main (int argc, char **argv)
 	cv::CascadeClassifier classifier(det_parameters.face_detector_location);
 	dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector();
 
+	// Loading the AU prediction models
+	string au_loc = "AU_predictors/AU_all_static.txt";
+
+	if (!boost::filesystem::exists(boost::filesystem::path(au_loc)))
+	{
+		boost::filesystem::path loc = boost::filesystem::path(arguments[0]).parent_path() / au_loc;
+
+		if (boost::filesystem::exists(loc))
+		{
+			au_loc = loc.string();
+		}
+		else
+		{
+			cout << "Can't find AU prediction files, exiting" << endl;
+			return 0;
+		}
+	}
+
+	// Used for image masking for AUs
+	string tri_loc;
+	if (boost::filesystem::exists(boost::filesystem::path("model/tris_68_full.txt")))
+	{
+		std::ifstream triangulation_file("model/tris_68_full.txt");
+		tri_loc = "model/tris_68_full.txt";
+	}
+	else
+	{
+		boost::filesystem::path loc = boost::filesystem::path(arguments[0]).parent_path() / "model/tris_68_full.txt";
+		tri_loc = loc.string();
+
+		if (!exists(loc))
+		{
+			cout << "Can't find triangulation files, exiting" << endl;
+			return 0;
+		}
+	}
+
+	FaceAnalysis::FaceAnalyser face_analyser(vector<cv::Vec3d>(), 0.7, 112, 112, au_loc, tri_loc);
+
 	bool visualise = !det_parameters.quiet_mode;
 
 	// Do some image loading
@@ -396,6 +473,8 @@ int main (int argc, char **argv)
 
 				}
 
+				auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false);
+
 				// Writing out the detected landmarks (in an OS independent manner)
 				if(!output_landmark_locations.empty())
 				{
@@ -412,7 +491,7 @@ int main (int argc, char **argv)
 					boost::filesystem::path fname = out_feat_path.filename().replace_extension("");
 					boost::filesystem::path ext = out_feat_path.extension();
 					string outfeatures = dir.string() + preferredSlash + fname.string() + string(name) + ext.string();
-					write_out_landmarks(outfeatures, clnf_model);
+					write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second);
 				}
 
 				if (!output_pose_locations.empty())
@@ -503,11 +582,13 @@ int main (int argc, char **argv)
 				FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false);
 			}
 
+			auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false);
+
 			// Writing out the detected landmarks
 			if(!output_landmark_locations.empty())
 			{
 				string outfeatures = output_landmark_locations.at(i);
-				write_out_landmarks(outfeatures, clnf_model);
+				write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second);
 			}
 
 			// Writing out the detected landmarks

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -324,7 +324,7 @@ int main (int argc, char **argv)
 	
 	// Used for image masking
 
-	cv::Mat_<int> triangulation;
+	cv::Mat_<int> triangulation;//TODO rem?
 	string tri_loc;
 	if(boost::filesystem::exists(path("model/tris_68_full.txt")))
 	{

lib/local/FaceAnalyser/AU_predictors/AU_all_best.txt

@@ -1,20 +1,18 @@
-svr_disfa/AU_1_static.dat AU01
-svr_disfa/AU_2_dyn.dat AU02
-svr_disfa/AU_4_static.dat AU04
-svm_combined/AU_4_dynamic_combined_all.dat AU04
-svr_disfa/AU_5_dyn.dat AU05
-svr_combined/AU_6_static_intensity_combined.dat AU06
-svr_disfa/AU_9_dyn.dat AU09
-svr_bp4d/AU_10_static_intensity.dat AU10
-svm_combined/AU_12_static_combined_all.dat AU12
-svr_combined/AU_12_static_intensity_combined.dat AU12
-svr_bp4d/AU_14_static_intensity.dat AU14
-svr_disfa/AU_15_dyn.dat AU15
-svm_combined/AU_15_dynamic_combined_all.dat AU15
-svr_combined/AU_17_static_intensity_combined.dat AU17
-svr_disfa/AU_20_dyn.dat AU20
-svm_bp4d/AU_23_static.dat AU23
-svr_disfa/AU_25_static.dat AU25
-svr_disfa/AU_26_dyn.dat AU26
-svm_semaine/AU_28_static.dat AU28
-svm_semaine/AU_45_dynamic.dat AU45
+svm_combined/AU_1_dynamic.dat AU01
+svm_combined/AU_2_dynamic.dat AU02
+svm_combined/AU_4_static.dat AU04
+svm_combined/AU_5_static.dat AU05
+svm_combined/AU_6_static.dat AU06
+svm_combined/AU_7_static.dat AU07
+svm_combined/AU_9_dynamic.dat AU09
+svm_combined/AU_10_static.dat AU10
+svm_combined/AU_12_static.dat AU12
+svm_combined/AU_14_static.dat AU14
+svm_combined/AU_15_dynamic.dat AU15
+svm_combined/AU_17_dynamic.dat AU17
+svm_combined/AU_20_dynamic.dat AU20
+svm_combined/AU_23_static.dat AU23
+svm_combined/AU_25_dynamic.dat AU25
+svm_combined/AU_26_dynamic.dat AU26
+svm_combined/AU_28_static.dat AU28
+svm_combined/AU_45_dynamic.dat AU45

lib/local/FaceAnalyser/AU_predictors/AU_all_static.txt

@@ -1,23 +1,35 @@
-svr_disfa/AU_1_static.dat AU01
-svr_disfa/AU_2_static.dat AU02
-svr_disfa/AU_4_static.dat AU04
-svm_combined/AU_4_static_combined_all.dat AU04
-svr_disfa/AU_5_static.dat AU05
-svr_combined/AU_6_static_intensity_combined.dat AU06
-svm_bp4d/AU_7_static.dat AU07
-svr_disfa/AU_9_static.dat AU09
-svr_bp4d/AU_10_static_intensity.dat AU10
-svm_combined/AU_12_static_combined_all.dat AU12
-svr_combined/AU_12_static_intensity_combined.dat AU12
-svr_bp4d/AU_14_static_intensity.dat AU14
-svr_disfa/AU_15_static.dat AU15
-svm_combined/AU_15_static_combined_all.dat AU15
-svr_combined/AU_17_static_intensity_combined.dat AU17
-svm_combined/AU_17_static_combined_all.dat AU17
-svr_disfa/AU_20_static.dat AU20
-svm_bp4d/AU_23_static.dat AU23
-svr_disfa/AU_25_static.dat AU25
-svm_combined/AU_25_static_combined_all.dat AU25
-svr_disfa/AU_26_static.dat AU26
-svm_semaine/AU_28_static.dat AU28
-svm_semaine/AU_45_static.dat AU45
+svm_combined/AU_1_static.dat AU01
+svm_combined/AU_2_static.dat AU02
+svm_combined/AU_4_static.dat AU04
+svm_combined/AU_5_static.dat AU05
+svm_combined/AU_6_static.dat AU06
+svm_combined/AU_7_static.dat AU07
+svm_combined/AU_9_static.dat AU09
+svm_combined/AU_10_static.dat AU10
+svm_combined/AU_12_static.dat AU12
+svm_combined/AU_14_static.dat AU14
+svm_combined/AU_15_static.dat AU15
+svm_combined/AU_17_static.dat AU17
+svm_combined/AU_20_static.dat AU20
+svm_combined/AU_23_static.dat AU23
+svm_combined/AU_25_static.dat AU25
+svm_combined/AU_26_static.dat AU26
+svm_combined/AU_28_static.dat AU28
+svm_combined/AU_45_static.dat AU45
+svr_combined/AU_1_static_intensity_comb.dat AU01
+svr_combined/AU_2_static_intensity_comb.dat AU02
+svr_combined/AU_4_static_intensity_comb.dat AU04
+svr_combined/AU_5_static_intensity.dat AU05
+svr_combined/AU_6_static_intensity_comb.dat AU06
+svr_combined/AU_7_static_intensity_comb.dat AU07
+svr_combined/AU_9_static_intensity.dat AU09
+svr_combined/AU_10_static_intensity_comb.dat AU10
+svr_combined/AU_12_static_intensity_comb.dat AU12
+svr_combined/AU_14_static_intensity.dat AU14
+svr_combined/AU_15_static_intensity_comb.dat AU15
+svr_combined/AU_17_static_intensity_comb.dat AU17
+svr_combined/AU_20_static_intensity.dat AU20
+svr_combined/AU_23_static_intensity_comb.dat AU23
+svr_combined/AU_25_static_intensity.dat AU25
+svr_combined/AU_26_static_intensity_comb.dat AU26
+svr_combined/AU_45_static_intensity_comb.dat AU45

lib/local/FaceAnalyser/include/FaceAnalyser.h

@@ -99,6 +99,10 @@ public:
 	std::vector<std::pair<std::string, double>> GetCurrentAUsReg() const;   // AU intensity
 	std::vector<std::pair<std::string, double>> GetCurrentAUsCombined() const; // Both presense and intensity
 
+	// A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity
+	// This call is useful for detecting action units in images
+	std::pair<std::vector<std::pair<string, double>>, std::vector<std::pair<string, double>>> PredictStaticAUs(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, bool visualise = true);
+
 	void Reset();
 
 	void GetLatestHOG(cv::Mat_<double>& hog_descriptor, int& num_rows, int& num_cols);
@@ -123,8 +127,8 @@ public:
 	std::vector<bool> GetDynamicAUReg() const; // Intensity
 
 
-	void ExtractAllPredictionsOfflineReg(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps);
-	void ExtractAllPredictionsOfflineClass(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps);
+	void ExtractAllPredictionsOfflineReg(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps, bool dynamic);
+	void ExtractAllPredictionsOfflineClass(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps, bool dynamic);
 
 private:
 

lib/local/FaceAnalyser/src/FaceAnalyser.cpp

@@ -303,6 +303,59 @@ void FaceAnalyser::ExtractCurrentMedians(vector<cv::Mat>& hog_medians, vector<cv
 	}
 }
 
+std::pair<std::vector<std::pair<string, double>>, std::vector<std::pair<string, double>>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, bool visualise)
+{
+	
+	// First align the face
+	AlignFaceMask(aligned_face, frame, clnf, triangulation, true, align_scale, align_width, align_height);
+	
+	// Extract HOG descriptor from the frame and convert it to a useable format
+	cv::Mat_<double> hog_descriptor;
+	Extract_FHOG_descriptor(hog_descriptor, aligned_face, this->num_hog_rows, this->num_hog_cols);
+
+	// Store the descriptor
+	hog_desc_frame = hog_descriptor;
+
+	cv::Vec3d curr_orient(clnf.params_global[1], clnf.params_global[2], clnf.params_global[3]);
+	int orientation_to_use = GetViewId(this->head_orientations, curr_orient);
+	
+	// Geom descriptor and its median
+	geom_descriptor_frame = clnf.params_local.t();
+
+	// Stack with the actual feature point locations (without mean)
+	cv::Mat_<double> locs = clnf.pdm.princ_comp * geom_descriptor_frame.t();
+
+	cv::hconcat(locs.t(), geom_descriptor_frame.clone(), geom_descriptor_frame);
+	
+	// First convert the face image to double representation as a row vector
+	cv::Mat_<uchar> aligned_face_cols(1, aligned_face.cols * aligned_face.rows * aligned_face.channels(), aligned_face.data, 1);
+	cv::Mat_<double> aligned_face_cols_double;
+	aligned_face_cols.convertTo(aligned_face_cols_double, CV_64F);
+
+	// Visualising the median HOG
+	if (visualise)
+	{
+		FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_visualisation);
+	}
+
+	// Perform AU prediction	
+	auto AU_predictions_intensity = PredictCurrentAUs(orientation_to_use);
+	auto AU_predictions_occurence = PredictCurrentAUsClass(orientation_to_use);
+
+	// Make sure intensity is within range (0-5)
+	for (size_t au = 0; au < AU_predictions_intensity.size(); ++au)
+	{
+		if (AU_predictions_intensity[au].second < 0)
+			AU_predictions_intensity[au].second = 0;
+
+		if (AU_predictions_intensity[au].second > 5)
+			AU_predictions_intensity[au].second = 5;
+	}
+
+	return std::pair<std::vector<std::pair<std::string, double>>, std::vector<std::pair<std::string, double>>>(AU_predictions_intensity, AU_predictions_occurence);
+
+}
+
 void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, double timestamp_seconds, bool online, bool visualise)
 {
 
@@ -607,7 +660,6 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string,
 	confidences = this->confidences;
 	successes = this->valid_preds;
 
-	// TODO only if the video is long enough or there is enough range? Compare stdev of BP4D and this
 	for(auto au_iter = AU_predictions_reg_all_hist.begin(); au_iter != AU_predictions_reg_all_hist.end(); ++au_iter)
 	{
 		vector<double> au_good;
@@ -665,6 +717,28 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string,
 		}
 	}
 
+	// Perform some prediction smoothing
+	for (auto au_iter = au_predictions.begin(); au_iter != au_predictions.end(); ++au_iter)
+	{
+		string au_name = au_iter->first;
+
+		// Perform a moving average of 3 frames
+		int window_size = 3;
+		vector<double> au_vals_tmp = au_iter->second;
+		for (size_t i = (window_size - 1) / 2; i < au_iter->second.size() - (window_size - 1) / 2; ++i)
+		{
+			double sum = 0;
+			for (int w = -(window_size - 1) / 2; w < (window_size - 1) / 2; ++w)
+			{
+				sum += au_vals_tmp[i + w];
+			}
+			sum = sum / window_size;
+
+			au_iter->second[i] = sum;
+		}
+
+	}
+
 
 }
 

matlab_runners/Action Unit Experiments/BP4D_valid_res_class.txt

@@ -1,11 +1,11 @@
-AU1 class, Precision - 0.426, Recall - 0.389, F1 - 0.406
-AU2 class, Precision - 0.326, Recall - 0.366, F1 - 0.345
-AU4 class, Precision - 0.465, Recall - 0.441, F1 - 0.453
-AU6 class, Precision - 0.759, Recall - 0.760, F1 - 0.760
-AU7 class, Precision - 0.741, Recall - 0.660, F1 - 0.698
-AU10 class, Precision - 0.863, Recall - 0.813, F1 - 0.837
-AU12 class, Precision - 0.882, Recall - 0.846, F1 - 0.864
-AU14 class, Precision - 0.575, Recall - 0.776, F1 - 0.660
-AU15 class, Precision - 0.348, Recall - 0.538, F1 - 0.423
-AU17 class, Precision - 0.575, Recall - 0.595, F1 - 0.585
-AU23 class, Precision - 0.395, Recall - 0.514, F1 - 0.447
+AU1 class, Precision - 0.470, Recall - 0.527, F1 - 0.497
+AU2 class, Precision - 0.371, Recall - 0.376, F1 - 0.373
+AU4 class, Precision - 0.422, Recall - 0.570, F1 - 0.485
+AU6 class, Precision - 0.845, Recall - 0.698, F1 - 0.765
+AU7 class, Precision - 0.719, Recall - 0.766, F1 - 0.742
+AU10 class, Precision - 0.811, Recall - 0.801, F1 - 0.806
+AU12 class, Precision - 0.902, Recall - 0.780, F1 - 0.837
+AU14 class, Precision - 0.513, Recall - 0.874, F1 - 0.647
+AU15 class, Precision - 0.406, Recall - 0.431, F1 - 0.418
+AU17 class, Precision - 0.638, Recall - 0.615, F1 - 0.626
+AU23 class, Precision - 0.357, Recall - 0.507, F1 - 0.419

matlab_runners/Action Unit Experiments/Bosphorus_res_class.txt

@@ -0,0 +1,17 @@
+AU1 class, Precision - 0.371, Recall - 0.756, F1 - 0.498
+AU2 class, Precision - 0.303, Recall - 0.829, F1 - 0.443
+AU4 class, Precision - 0.552, Recall - 0.856, F1 - 0.671
+AU5 class, Precision - 0.361, Recall - 0.878, F1 - 0.511
+AU6 class, Precision - 0.348, Recall - 0.817, F1 - 0.488
+AU7 class, Precision - 0.819, Recall - 0.717, F1 - 0.765
+AU9 class, Precision - 0.361, Recall - 0.946, F1 - 0.522
+AU10 class, Precision - 0.336, Recall - 0.780, F1 - 0.469
+AU12 class, Precision - 0.687, Recall - 0.856, F1 - 0.762
+AU14 class, Precision - 0.190, Recall - 0.863, F1 - 0.311
+AU15 class, Precision - 0.156, Recall - 0.844, F1 - 0.263
+AU17 class, Precision - 0.284, Recall - 0.866, F1 - 0.428
+AU20 class, Precision - 0.130, Recall - 0.930, F1 - 0.228
+AU23 class, Precision - 0.103, Recall - 0.837, F1 - 0.183
+AU25 class, Precision - 0.839, Recall - 0.829, F1 - 0.834
+AU26 class, Precision - 0.363, Recall - 0.794, F1 - 0.498
+AU45 class, Precision - 0.377, Recall - 0.842, F1 - 0.521

matlab_runners/Action Unit Experiments/SEMAINE_valid_res.txt

@@ -1,6 +1,6 @@
-AU2 class, Precision - 0.489, Recall - 0.536, F1 - 0.511
-AU12 class, Precision - 0.521, Recall - 0.751, F1 - 0.615
-AU17 class, Precision - 0.408, Recall - 0.486, F1 - 0.444
-AU25 class, Precision - 0.410, Recall - 0.542, F1 - 0.467
-AU28 class, Precision - 0.493, Recall - 0.398, F1 - 0.441
-AU45 class, Precision - 0.223, Recall - 0.723, F1 - 0.341
+AU2 class, Precision - 0.366, Recall - 0.718, F1 - 0.484
+AU12 class, Precision - 0.434, Recall - 0.790, F1 - 0.560
+AU17 class, Precision - 0.129, Recall - 0.830, F1 - 0.223
+AU25 class, Precision - 0.362, Recall - 0.580, F1 - 0.446
+AU28 class, Precision - 0.362, Recall - 0.514, F1 - 0.425
+AU45 class, Precision - 0.295, Recall - 0.571, F1 - 0.389

matlab_runners/Action Unit Experiments/helpers/extract_Bosphorus_labels.m

@@ -0,0 +1,103 @@
+function [ labels, valid_ids, filenames  ] = extract_Bosphorus_labels( Bosphorus_dir, recs, aus )
+%EXTRACT_SEMAINE_LABELS Summary of this function goes here
+%   Detailed explanation goes here
+    
+    % Ignoring rare ones or ones that don't overlap with other datasets
+    aus_Bosphorus = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 43];
+    aus(aus == 45) = 43;
+    
+    %%
+    fid = fopen([Bosphorus_dir, './facscodes/facscodes.lst']);
+    % Skipping the header
+    fgetl(fid);
+    fgetl(fid);
+    
+    % Starting to read
+    data = fgetl(fid);
+    
+    all_aus = [];
+    valid = [];
+    
+    id = 1;
+
+    filenames = {};
+    
+    while ischar(data)
+        
+        d = strsplit(data, '->');
+        data = fgetl(fid);
+        
+        filename = strtrim(d{1});
+                        
+        % Skip extreme poses
+        if(~isempty(findstr(filename, 'CR')) || ~isempty(findstr(filename, 'YR') > 0) || ~isempty(findstr(filename, 'PR_U'))|| ~isempty(findstr(filename, 'PR_D')))
+           continue; 
+        end
+        
+        % ignore labels from non requested users
+        if(isempty(strmatch(filename(1:5), recs)))
+            continue;
+        end
+        
+        filenames = cat(1, filenames, filename);
+        
+        aus_str = d{2}(3:end);
+
+        % decode the AU data
+        aus_c = strsplit(aus_str, '+');
+        
+        curr_img_au = zeros(1, 80);
+        
+        for i=1:numel(aus_c)
+            
+            if(aus_c{i} == '0')
+                
+                continue
+            end
+            
+            intensity = -1;
+            
+            intensity_str = aus_c{i}(end);
+            if(intensity_str == 'A')
+                intensity = 1;
+            elseif(intensity_str == 'B')
+                intensity = 2;               
+            elseif(intensity_str == 'C')
+                intensity = 3;               
+            elseif(intensity_str == 'D')
+                intensity = 4;               
+            elseif(intensity_str == 'E')
+                intensity = 5;               
+            end
+            
+            if(~isempty(str2num(aus_c{i}(1))))
+                if(intensity ~= -1)
+                    num = str2num(aus_c{i}(1:end-1));
+                else
+                    num = str2num(aus_c{i}(1:end));
+                    intensity = 3; % if no intensity given just assume 3
+                end
+            else
+                if(intensity ~= -1)
+                    num = str2num(aus_c{i}(2:end-1));
+                else
+                    num = str2num(aus_c{i}(2:end));
+                    intensity = 3; % if no intensity given just assume 3
+                end
+            end
+                                    
+            curr_img_au(1, num) = intensity;
+        end
+        all_aus = cat(1, all_aus, curr_img_au);
+        valid = cat(1, valid, [true]);
+        
+        id = id + 1;
+
+    end
+    %aus_bosph = dlmread(, '->', 3, 0);
+    fclose(fid);
+    
+    valid_ids = logical(valid);
+    labels = all_aus(:, aus);
+end
+

matlab_runners/Action Unit Experiments/helpers/find_Bosphorus.m

@@ -0,0 +1,21 @@
+if(exist('D:/Datasets/Bosphorus/', 'file'))
+    Bosphorus_dir = 'D:\Datasets\Bosphorus/';   
+else
+    fprintf('Bosphorus dataset location not found (or not defined)\n'); 
+end
+
+hog_data_dir = ['D:\Datasets\face_datasets'];
+
+all_recs = dir([Bosphorus_dir, '/BosphorusDB/BosphorusDB/bs*']);
+all_recs_mat = cat(1, all_recs.name);
+all_recs = cell(numel(all_recs), 1);
+
+for i=1:size(all_recs_mat,1)
+   
+    all_recs{i} = all_recs_mat(i,:);
+    
+end
+
+devel_recs = all_recs(1:3:end);
+train_recs = setdiff(all_recs, devel_recs);
+

matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.m

@@ -13,32 +13,56 @@ executable = '"../../x64/Release/FeatureExtraction.exe"';
 
 bp4d_dirs = {'F002', 'F004', 'F006', 'F008', 'F010', 'F012', 'F014', 'F016', 'F018', 'F020', 'F022', 'M002', 'M004', 'M006', 'M008', 'M010', 'M012', 'M014', 'M016', 'M018'};
 
-parfor f1=1:numel(bp4d_dirs)
+%% Before running BP4D convert it to a smaller format and move each person to the same directory
+% This is done so that dynamic models would work on it as otherwise the
+% clips are a bit too short
 
-    if(isdir([bp4d_loc, bp4d_dirs{f1}]))
+new_bp4d_dirs = {};
+
+% This might take some time
+for i = 1:numel(bp4d_dirs)
+    dirs = dir([bp4d_loc, '/', bp4d_dirs{i}, '/T*']);
+    tmp_dir = [bp4d_loc, '/', bp4d_dirs{i}, '/tmp/'];
+    new_bp4d_dirs = cat(1, new_bp4d_dirs, tmp_dir);
+    
+    if(~exist(tmp_dir, 'file'))
+        mkdir(tmp_dir);
         
-        bp4d_2_dirs = dir([bp4d_loc, bp4d_dirs{f1}]);
-        bp4d_2_dirs = bp4d_2_dirs(3:end);
-        
-        f1_dir = bp4d_dirs{f1};
-
-        command = [executable ' -asvid -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze '];
-
-        for f2=1:numel(bp4d_2_dirs)
-            f2_dir = bp4d_2_dirs(f2).name;
-            if(isdir([bp4d_loc, bp4d_dirs{f1}]))
+        % Move all images and resize them
+        for d=1:numel(dirs)
+           
+            in_files = dir([bp4d_loc, '/', bp4d_dirs{i}, '/', dirs(d).name, '/*.jpg']);
+            
+            for img_ind=1:numel(in_files)
+               
+                img_file = [bp4d_loc, '/', bp4d_dirs{i}, '/', dirs(d).name, '/', in_files(img_ind).name];
+                img = imread(img_file);
+                img = imresize(img, 0.5);
+                img_out = [tmp_dir, dirs(d).name, '_', in_files(img_ind).name];
+                imwrite(img, img_out);
                 
-                curr_vid = [bp4d_loc, f1_dir, '/', f2_dir, '/'];
-
-                name = [f1_dir '_' f2_dir];
-                output_file = [out_loc name '.au.txt'];
-                
-                command = cat(2, command, [' -fdir "' curr_vid '" -of "' output_file '"']);
             end
+            
         end
         
-        dos(command);
     end
+    
+end
+%%
+
+parfor f1=1:numel(new_bp4d_dirs)
+    % TODO rem - attempt a static model
+    command = [executable ' -asvid -no2Dfp -no3Dfp -noMparams -noPose -noGaze '];
+
+    [f,~,~] = fileparts(new_bp4d_dirs{f1});
+    [f,~,~] = fileparts(f);
+    [~,f,~] = fileparts(f);
+    output_file = [out_loc f '.au.txt'];
+
+    command = cat(2, command, [' -fdir "' new_bp4d_dirs{f1} '" -of "' output_file '"']);
+
+    dos(command);
+
 end
 
 %%
@@ -52,7 +76,7 @@ aus_BP4D = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
 labels_gt = cat(1, labels_gt{:});
 
 %% Identifying which column IDs correspond to which AU
-tab = readtable([out_loc, bp4d_dirs{1}, '_T1.au.txt']);
+tab = readtable([out_loc, bp4d_dirs{1}, '.au.txt']);
 column_names = tab.Properties.VariableNames;
 
 % As there are both classes and intensities list and evaluate both of them
@@ -92,9 +116,13 @@ end
 preds_all_class = [];
 preds_all_int = [];
 
-for i=1:numel(filenames)
+for i=1:numel(new_bp4d_dirs)
    
-    fname = [out_loc, filenames{i}, '.au.txt'];
+    [f,~,~] = fileparts(new_bp4d_dirs{i});
+    [f,~,~] = fileparts(f);
+    [~,f,~] = fileparts(f);
+    
+    fname = [out_loc, f, '.au.txt'];
     preds = dlmread(fname, ',', 1, 0);
     
     % Read all of the intensity AUs

matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.asv

@@ -0,0 +1,189 @@
+% Perform static model prediction using images
+
+clear
+
+addpath('./helpers');
+
+find_Bosphorus;
+out_loc = './out_bosph/';
+
+if(~exist(out_loc, 'dir'))
+    mkdir(out_loc);
+end
+
+%%
+executable = '"../../x64/Release/FaceLandmarkImg.exe"';
+
+bosph_dirs = dir([Bosphorus_dir, '/BosphorusDB/BosphorusDB/bs*']);
+
+%%
+parfor f1=1:numel(bosph_dirs)
+
+    command = executable;
+
+    input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name];
+    command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']);
+    command = cat(2, command, ' -multi_view 1 -wild');
+
+    dos(command);
+
+end
+
+%%
+
+aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45];
+
+[ labels_gt, valid_ids, filenames] = extract_Bosphorus_labels(Bosphorus_dir, all_recs, aus_Bosph);
+
+%% Read the predicted values
+
+% First read the first file to get the ids and line numbers
+% au occurences
+fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
+data = fgetl(fid);
+
+ind = 0;
+beg_ind = -1;
+end_ind = -1;
+aus_det = [];
+aus_det_id = [];
+
+while ischar(data)
+    if(~isempty(findstr(data, 'au occurences:')))
+        num_occurences = str2num(data(numel('au occurences:')+1:end));
+        % Skip ahead two lines
+        data = fgetl(fid);   
+        data = fgetl(fid);   
+        ind = ind + 2;
+        beg_ind = ind;
+    end
+    
+    if(beg_ind ~= -1 && end_ind == -1)
+        if(~isempty(findstr(data, '}')))
+            end_ind = ind;
+        else
+            d = strsplit(data, ' '); 
+            aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
+            aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
+        end
+    end
+    
+    data = fgetl(fid);
+    ind = ind + 1;
+end
+fclose(fid);
+
+%%
+labels_pred = zeros(size(labels_gt));
+for i=1:numel(filenames)
+
+    % Will need to read the relevant AUs only
+    if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
+        fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
+        for k=1:beg_ind
+            data = fgetl(fid);
+        end
+
+        for k=1:num_occurences
+            data = fgetl(fid);
+            if(sum(aus_Bosph == aus_det(k))>0)
+                d = strsplit(data, ' '); 
+                labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
+            end
+        end
+
+        fclose(fid);
+    end
+end
+
+
+%%
+
+
+%%
+f = fopen('Bosphorus_res_class.txt', 'w');
+for au = 1:numel(aus_Bosph)
+
+    if(inds_au_int(au) ~= 0)
+        tp = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) >= 1);
+        fp = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) >= 1);
+        fn = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) < 1);
+        tn = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) < 1);
+
+        precision = tp./(tp+fp);
+        recall = tp./(tp+fn);
+
+        f1 = 2 * precision .* recall ./ (precision + recall);
+
+        fprintf(f, 'AU%d intensity, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
+    end
+    
+    if(inds_au_class(au) ~= 0)
+        tp = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 1);
+        fp = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 1);
+        fn = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 0);
+        tn = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 0);
+
+        precision = tp./(tp+fp);
+        recall = tp./(tp+fn);
+
+        f1 = 2 * precision .* recall ./ (precision + recall);
+
+        fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
+    end    
+    
+end
+fclose(f);
+
+%%
+addpath('./helpers/');
+
+find_BP4D;
+
+aus_BP4D = [6, 10, 12, 14, 17];
+[ labels_gt, valid_ids, vid_ids, filenames] = extract_BP4D_labels_intensity(BP4D_dir_int, devel_recs, aus_BP4D);
+labels_gt = cat(1, labels_gt{:});
+
+%% Identifying which column IDs correspond to which AU
+tab = readtable([out_loc, bosph_dirs{1}, '_T1.au.txt']);
+column_names = tab.Properties.VariableNames;
+
+% As there are both classes and intensities list and evaluate both of them
+aus_pred_int = [];
+inds_int_in_file = [];
+
+for c=1:numel(column_names)
+    if(strfind(column_names{c}, '_r') > 0)
+        aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2))));
+        inds_int_in_file = cat(1, inds_int_in_file, c);
+    end
+end
+
+%%
+inds_au_int = zeros(size(aus_BP4D));
+
+for ind=1:numel(aus_BP4D)  
+    if(~isempty(find(aus_pred_int==aus_BP4D(ind), 1)))
+        inds_au_int(ind) = find(aus_pred_int==aus_BP4D(ind));
+    end
+end
+
+preds_all_int = [];
+
+for i=1:numel(filenames)
+   
+    fname = [out_loc, filenames{i}, '.au.txt'];
+    preds = dlmread(fname, ',', 1, 0);
+    
+    % Read all of the intensity AUs
+    preds_int = preds(:, inds_int_in_file);    
+    preds_all_int = cat(1, preds_all_int, preds_int);
+end
+
+%%
+f = fopen('BP4D_valid_res_int.txt', 'w');
+for au = 1:numel(aus_BP4D)
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_au_prediction_results( preds_all_int(:, inds_au_int(au)), labels_gt(:,au));
+    fprintf(f, 'AU%d results - corr %.3f, ccc - %.3f\n', aus_BP4D(au), corrs, ccc);    
+end
+fclose(f);

matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m

@@ -0,0 +1,200 @@
+% Perform static model prediction using images
+
+clear
+
+addpath('./helpers');
+
+find_Bosphorus;
+out_loc = './out_bosph/';
+
+if(~exist(out_loc, 'dir'))
+    mkdir(out_loc);
+end
+
+%%
+executable = '"../../x64/Release/FaceLandmarkImg.exe"';
+
+bosph_dirs = dir([Bosphorus_dir, '/BosphorusDB/BosphorusDB/bs*']);
+
+%%
+parfor f1=1:numel(bosph_dirs)
+
+    command = executable;
+
+    input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name];
+    command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']);
+    command = cat(2, command, ' -multi_view 1 -wild');
+
+    dos(command);
+
+end
+
+%%
+
+aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45];
+
+[ labels_gt, valid_ids, filenames] = extract_Bosphorus_labels(Bosphorus_dir, all_recs, aus_Bosph);
+
+%% Read the predicted values
+
+% First read the first file to get the ids and line numbers
+% au occurences
+fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
+data = fgetl(fid);
+
+ind = 0;
+beg_ind = -1;
+end_ind = -1;
+aus_det = [];
+aus_det_id = [];
+
+while ischar(data)
+    if(~isempty(findstr(data, 'au occurences:')))
+        num_occurences = str2num(data(numel('au occurences:')+1:end));
+        % Skip ahead two lines
+        data = fgetl(fid);   
+        data = fgetl(fid);   
+        ind = ind + 2;
+        beg_ind = ind;
+    end
+    
+    if(beg_ind ~= -1 && end_ind == -1)
+        if(~isempty(findstr(data, '}')))
+            end_ind = ind;
+        else
+            d = strsplit(data, ' '); 
+            aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
+            aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
+        end
+    end
+    
+    data = fgetl(fid);
+    ind = ind + 1;
+end
+fclose(fid);
+
+%%
+labels_pred = zeros(size(labels_gt));
+for i=1:numel(filenames)
+
+    % Will need to read the relevant AUs only
+    if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
+        fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
+        for k=1:beg_ind
+            data = fgetl(fid);
+        end
+
+        for k=1:num_occurences
+            data = fgetl(fid);
+            if(sum(aus_Bosph == aus_det(k))>0)
+                d = strsplit(data, ' '); 
+                labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
+            end
+        end
+
+        fclose(fid);
+    end
+end
+
+%%
+f = fopen('Bosphorus_res_class.txt', 'w');
+labels_gt_bin = labels_gt;
+labels_gt_bin(labels_gt_bin > 1) = 1;
+for au = 1:numel(aus_Bosph)
+  
+    tp = sum(labels_gt_bin(:,au) == 1 & labels_pred(:, au) == 1);
+    fp = sum(labels_gt_bin(:,au) == 0 & labels_pred(:, au) == 1);
+    fn = sum(labels_gt_bin(:,au) == 1 & labels_pred(:, au) == 0);
+    tn = sum(labels_gt_bin(:,au) == 0 & labels_pred(:, au) == 0);
+
+    precision = tp./(tp+fp);
+    recall = tp./(tp+fn);
+
+    f1 = 2 * precision .* recall ./ (precision + recall);
+
+    fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_Bosph(au), precision, recall, f1);
+
+end
+fclose(f);
+
+%% Read the predicted values for intensities
+
+% First read the first file to get the ids and line numbers
+% au occurences
+fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
+data = fgetl(fid);
+
+ind = 0;
+beg_ind = -1;
+end_ind = -1;
+aus_det = [];
+aus_det_id = [];
+
+while ischar(data)
+    if(~isempty(findstr(data, 'au intensities:')))
+        num_occurences = str2num(data(numel('au intensities:')+1:end));
+        % Skip ahead two lines
+        data = fgetl(fid);   
+        data = fgetl(fid);   
+        ind = ind + 2;
+        beg_ind = ind;
+    end
+    
+    if(beg_ind ~= -1 && end_ind == -1)
+        if(~isempty(findstr(data, '}')))
+            end_ind = ind;
+        else
+            d = strsplit(data, ' '); 
+            aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
+            aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
+        end
+    end
+    
+    data = fgetl(fid);
+    ind = ind + 1;
+end
+fclose(fid);
+
+%%
+labels_pred = zeros(size(labels_gt));
+for i=1:numel(filenames)
+
+    % Will need to read the relevant AUs only
+    if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
+        fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
+        for k=1:beg_ind
+            data = fgetl(fid);
+        end
+
+        for k=1:num_occurences
+            data = fgetl(fid);
+            if(sum(aus_Bosph == aus_det(k))>0)
+                d = strsplit(data, ' '); 
+                labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
+            end
+        end
+
+        fclose(fid);
+    end
+end
+
+%%
+f = fopen('Bosphorus_res_class.txt', 'w');
+labels_gt_bin = labels_gt;
+labels_gt_bin(labels_gt_bin > 1) = 1;
+for au = 1:numel(aus_Bosph)
+  
+    tp = sum(labels_gt_bin(:,au) == 1 & labels_pred(:, au) == 1);
+    fp = sum(labels_gt_bin(:,au) == 0 & labels_pred(:, au) == 1);
+    fn = sum(labels_gt_bin(:,au) == 1 & labels_pred(:, au) == 0);
+    tn = sum(labels_gt_bin(:,au) == 0 & labels_pred(:, au) == 0);
+
+    precision = tp./(tp+fp);
+    recall = tp./(tp+fn);
+
+    f1 = 2 * precision .* recall ./ (precision + recall);
+
+    fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_Bosph(au), precision, recall, f1);
+
+end
+fclose(f);

matlab_version/AU_training/data extraction/extract_Bosphorus_labels.m

@@ -0,0 +1,103 @@
+function [ labels, valid_ids, filenames  ] = extract_Bosphorus_labels( Bosphorus_dir, recs, aus )
+%EXTRACT_SEMAINE_LABELS Summary of this function goes here
+%   Detailed explanation goes here
+    
+    % Ignoring rare ones or ones that don't overlap with other datasets
+    aus_Bosphorus = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 43];
+    aus(aus == 45) = 43;
+    
+    %%
+    fid = fopen([Bosphorus_dir, './facscodes/facscodes.lst']);
+    % Skipping the header
+    fgetl(fid);
+    fgetl(fid);
+    
+    % Starting to read
+    data = fgetl(fid);
+    
+    all_aus = [];
+    valid = [];
+    
+    id = 1;
+
+    filenames = {};
+    
+    while ischar(data)
+        
+        d = strsplit(data, '->');
+        data = fgetl(fid);
+        
+        filename = strtrim(d{1});
+                        
+        % Skip extreme poses
+        if(~isempty(findstr(filename, 'CR')) || ~isempty(findstr(filename, 'YR') > 0) || ~isempty(findstr(filename, 'PR_U'))|| ~isempty(findstr(filename, 'PR_D')))
+           continue; 
+        end
+        
+        % ignore labels from non requested users
+        if(isempty(strmatch(filename(1:5), recs)))
+            continue;
+        end
+        
+        filenames = cat(1, filenames, filename);
+        
+        aus_str = d{2}(3:end);
+
+        % decode the AU data
+        aus_c = strsplit(aus_str, '+');
+        
+        curr_img_au = zeros(1, 80);
+        
+        for i=1:numel(aus_c)
+            
+            if(aus_c{i} == '0')
+                
+                continue
+            end
+            
+            intensity = -1;
+            
+            intensity_str = aus_c{i}(end);
+            if(intensity_str == 'A')
+                intensity = 1;
+            elseif(intensity_str == 'B')
+                intensity = 2;               
+            elseif(intensity_str == 'C')
+                intensity = 3;               
+            elseif(intensity_str == 'D')
+                intensity = 4;               
+            elseif(intensity_str == 'E')
+                intensity = 5;               
+            end
+            
+            if(~isempty(str2num(aus_c{i}(1))))
+                if(intensity ~= -1)
+                    num = str2num(aus_c{i}(1:end-1));
+                else
+                    num = str2num(aus_c{i}(1:end));
+                    intensity = 3; % if no intensity given just assume 3
+                end
+            else
+                if(intensity ~= -1)
+                    num = str2num(aus_c{i}(2:end-1));
+                else
+                    num = str2num(aus_c{i}(2:end));
+                    intensity = 3; % if no intensity given just assume 3
+                end
+            end
+                                    
+            curr_img_au(1, num) = intensity;
+        end
+        all_aus = cat(1, all_aus, curr_img_au);
+        valid = cat(1, valid, [true]);
+        
+        id = id + 1;
+
+    end
+    %aus_bosph = dlmread(, '->', 3, 0);
+    fclose(fid);
+    
+    valid_ids = logical(valid);
+    labels = all_aus(:, aus);
+end
+

matlab_version/AU_training/data extraction/extract_features_Bosphorus.m

@@ -0,0 +1,32 @@
+clear
+features_exe = '"..\..\..\x64\Release\FeatureExtraction.exe"';
+
+bosph_loc = 'D:\Datasets\Bosphorus\BosphorusDB\BosphorusDB/';
+
+out_loc = 'D:\Datasets\face_datasets/';
+
+% Go two levels deep
+bosph_dirs = dir([bosph_loc, '/bs*']);
+
+for f1=1:numel(bosph_dirs)
+        
+    name = [bosph_dirs(f1).name];
+
+    curr_vids = dir([bosph_loc, '/' name, '/*.png']);
+
+    parfor i=1:numel(curr_vids)
+        command = features_exe;
+        input_file = [bosph_loc, '/' name '/', curr_vids(i).name];
+        [~, curr_name, ~] = fileparts(curr_vids(i).name);
+        output_file = [out_loc, '/hog_aligned_rigid_b/',  curr_name, '/'];
+
+        output_hog = [out_loc, '/hog_aligned_rigid_b/',curr_name '.hog'];
+        output_params = [out_loc, '/model_params_b/', curr_name '.txt'];
+
+        command = cat(2, command, [' -rigid -f "' input_file '" -simalign "' output_file  '" -simscale 0.7 -simsize 112 ']);
+        command = cat(2, command, [' -hogalign "' output_hog '"' ' -of "' output_params ]);
+        command = cat(2, command, ['" -no2Dfp -no3Dfp -noAUs -noPose -noGaze -q ']);
+        dos(command);
+    end
+                
+end

matlab_version/AU_training/data extraction/extract_features_UNBC.m

@@ -9,10 +9,6 @@ out_loc = 'D:\Datasets\face_datasets/';
 unbc_dirs = dir(unbc_loc);
 unbc_dirs = unbc_dirs(3:end);
 
-if(~exist([out_loc, '/clm_params/'], 'file'))
-    mkdir([out_loc, '/clm_params/']);   
-end
-
 parfor f1=1:numel(unbc_dirs)
 
     unbc_dirs_level_2 = dir([unbc_loc, unbc_dirs(f1).name]);

matlab_version/AU_training/data extraction/find_Bosphorus.m

@@ -0,0 +1,21 @@
+if(exist('D:/Datasets/Bosphorus/', 'file'))
+    Bosphorus_dir = 'D:\Datasets\Bosphorus/';   
+else
+    fprintf('Bosphorus dataset location not found (or not defined)\n'); 
+end
+
+hog_data_dir = ['D:\Datasets\face_datasets'];
+
+all_recs = dir([Bosphorus_dir, '/BosphorusDB/BosphorusDB/bs*']);
+all_recs_mat = cat(1, all_recs.name);
+all_recs = cell(numel(all_recs), 1);
+
+for i=1:size(all_recs_mat,1)
+   
+    all_recs{i} = all_recs_mat(i,:);
+    
+end
+
+devel_recs = all_recs(1:3:end);
+train_recs = setdiff(all_recs, devel_recs);
+

matlab_version/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic.m

@@ -75,9 +75,21 @@ labels_devel = cat(1, labels_devel{:});
 
 valid_ids_test = valid_ids_devel_hog;
 
-% normalise the data
-load(pca_file);
-     
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+
 PC_n = zeros(size(PC)+size(train_geom_data, 2));
 PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
 PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(size(train_geom_data, 2));

matlab_version/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic_dynamic.m

@@ -71,9 +71,21 @@ devel_appearance_data = cat(2, devel_appearance_data, devel_geom_data);
 
 valid_ids_devel = valid_ids_devel_hog;
 
-% normalise the data
-load(pca_file);
-     
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+
 PC_n = zeros(size(PC)+size(train_geom_data, 2));
 PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
 PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(size(train_geom_data, 2));

matlab_version/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic_intensity.m

@@ -1,5 +1,5 @@
 function [data_train, labels_train, vid_ids_train_string, data_devel, labels_devel, vid_ids_devel_string, raw_devel, PC, means_norm, stds_norm, success_devel] = ...
-    Prepare_HOG_AU_data_generic_intensity(train_users, devel_users, au_train, bp4d_dir, hog_data_dir, pca_file)
+    Prepare_HOG_AU_data_generic_intensity(train_users, devel_users, au_train, bp4d_dir, hog_data_dir)
 
 %%
 addpath(genpath('../data extraction/'));
@@ -76,8 +76,20 @@ labels_devel = cat(1, labels_devel{:});
 
 success_devel = valid_ids_devel;
 
-% normalise the data
-load(pca_file);
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
 
 PC_n = zeros(size(PC)+size(devel_geom_data, 2));
 PC_n(1:size(PC,1), 1:size(PC,2)) = PC;

matlab_version/AU_training/experiments/BP4D/Script_HOG_SVR_dynamic_shift.m

@@ -14,10 +14,10 @@ hyperparams.p = 10.^(-2);
 hyperparams.validate_params = {'c', 'p'};
 
 % Set the training function
-svr_train = @svr_train_linear_shift_fancy;
+svr_train = @svr_train_linear_shift;
     
 % Set the test function (the first output will be used for validation)
-svr_test = @svr_test_linear_shift_fancy;
+svr_test = @svr_test_linear_shift;
 
 pca_loc = '../../pca_generation/generic_face_rigid.mat';
 

matlab_version/AU_training/experiments/BP4D/Script_HOG_SVR_static_shift.m

@@ -14,10 +14,10 @@ hyperparams.p = 10.^(-2);
 hyperparams.validate_params = {'c', 'p'};
 
 % Set the training function
-svr_train = @svr_train_linear_shift_fancy;
+svr_train = @svr_train_linear_shift;
     
 % Set the test function (the first output will be used for validation)
-svr_test = @svr_test_linear_shift_fancy;
+svr_test = @svr_test_linear_shift;
 
 pca_loc = '../../pca_generation/generic_face_rigid.mat';