Towards AU post-processing in the GUI.

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -234,9 +234,6 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
 	cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
 	const FaceAnalysis::FaceAnalyser& face_analyser);
 
-void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic);
-
-
 int main (int argc, char **argv)
 {
 
@@ -498,9 +495,7 @@ int main (int argc, char **argv)
 			catch(cv::Exception e)
 			{
 				WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
-			}
-
-			
+			}			
 		}
 
 		int frame_count = 0;
@@ -700,7 +695,7 @@ int main (int argc, char **argv)
 		if(output_files.size() > 0 && output_AUs)
 		{
 			cout << "Postprocessing the Action Unit predictions" << endl;
-			post_process_output_file(face_analyser, output_files[f_n], dynamic);
+			face_analyser.PostprocessOutputFile(output_files[f_n], dynamic);
 		}
 		// Reset the models for the next video
 		face_analyser.Reset();
@@ -724,124 +719,6 @@ int main (int argc, char **argv)
 	return 0;
 }
 
-// Allows for post processing of the AU signal
-void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic)
-{
-
-	vector<double> certainties;
-	vector<bool> successes;
-	vector<double> timestamps;
-	vector<std::pair<std::string, vector<double>>> predictions_reg;
-	vector<std::pair<std::string, vector<double>>> predictions_class;
-
-	// Construct the new values to overwrite the output file with
-	face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic);
-	face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic);
-
-	int num_class = predictions_class.size();
-	int num_reg = predictions_reg.size();
-
-	// Extract the indices of writing out first
-	vector<string> au_reg_names = face_analyser.GetAURegNames();
-	std::sort(au_reg_names.begin(), au_reg_names.end());
-	vector<int> inds_reg;
-
-	// write out ar the correct index
-	for (string au_name : au_reg_names)
-	{
-		for (int i = 0; i < num_reg; ++i)
-		{
-			if (au_name.compare(predictions_reg[i].first) == 0)
-			{
-				inds_reg.push_back(i);
-				break;
-			}
-		}
-	}
-
-	vector<string> au_class_names = face_analyser.GetAUClassNames();
-	std::sort(au_class_names.begin(), au_class_names.end());
-	vector<int> inds_class;
-
-	// write out ar the correct index
-	for (string au_name : au_class_names)
-	{
-		for (int i = 0; i < num_class; ++i)
-		{
-			if (au_name.compare(predictions_class[i].first) == 0)
-			{
-				inds_class.push_back(i);
-				break;
-			}
-		}
-	}
-	// Read all of the output file in
-	vector<string> output_file_contents;
-
-	std::ifstream infile(output_file);
-	string line;
-
-	while (std::getline(infile, line))
-		output_file_contents.push_back(line);
-
-	infile.close();
-
-	// Read the header and find all _r and _c parts in a file and use their indices
-	std::vector<std::string> tokens;
-	boost::split(tokens, output_file_contents[0], boost::is_any_of(","));
-
-	int begin_ind = -1;
-	
-	for (size_t i = 0; i < tokens.size(); ++i)
-	{
-		if (tokens[i].find("AU") != string::npos && begin_ind == -1)
-		{
-			begin_ind = i;
-			break;
-		}
-	}
-	int end_ind = begin_ind + num_class + num_reg;
-
-	// Now overwrite the whole file
-	std::ofstream outfile(output_file, ios_base::out);
-	// Write the header
-	outfile << std::setprecision(4);
-	outfile << output_file_contents[0].c_str() << endl;
-	
-	// Write the contents
-	for (int i = 1; i < (int)output_file_contents.size(); ++i)
-	{
-		std::vector<std::string> tokens;
-		boost::split(tokens, output_file_contents[i], boost::is_any_of(","));
-
-		boost::trim(tokens[0]);
-		outfile << tokens[0];
-
-		for (int t = 1; t < (int)tokens.size(); ++t)
-		{
-			if (t >= begin_ind && t < end_ind)
-			{
-				if(t - begin_ind < num_reg)
-				{
-					outfile << ", " << predictions_reg[inds_reg[t - begin_ind]].second[i - 1];
-				}
-				else
-				{
-					outfile << ", " << predictions_class[inds_class[t - begin_ind - num_reg]].second[i - 1];
-				}
-			}
-			else
-			{
-				boost::trim(tokens[t]);
-				outfile << ", " << tokens[t];
-			}
-		}
-		outfile << endl;
-	}
-		
-
-}
-
 void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
 	int num_landmarks, int num_eye_lmks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg)

gui/OpenFaceOffline/MainWindow.xaml.cs

@@ -420,7 +420,7 @@ namespace OpenFaceOffline
 
             // Setup the recorder first
             recorder = new Recorder(record_root, output_file_name, capture.width, capture.height, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose,
-                RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy);
+                RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy, DynamicAUModels);
 
             int frame_id = 0;
 

gui/OpenFaceOffline/Recorder.cs

@@ -20,9 +20,13 @@ namespace OpenFaceOffline
         List<string> au_reg_names;
         List<string> au_class_names;
 
+        String out_filename;
+
+        bool dynamic_AU_model;
+
         public Recorder(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, 
             bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG,
-            CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy)
+            CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model)
         {
 
             this.output_2D_landmarks = output_2D_landmarks; this.output_3D_landmarks = output_3D_landmarks;
@@ -32,12 +36,14 @@ namespace OpenFaceOffline
 
             this.fx = fx; this.fy = fy; this.cx = cx; this.cy = cy;
 
+            this.dynamic_AU_model = dynamic_AU_model;
+
             if (!System.IO.Directory.Exists(root))
             {
                 System.IO.Directory.CreateDirectory(root);
             }
-
-            output_features_file = new StreamWriter(root + "/" + filename + ".txt");
+            out_filename = root + "/" + filename + ".csv";
+            output_features_file = new StreamWriter(out_filename);
             output_features_file.Write("frame, timestamp, confidence, success");
 
             if (output_gaze)
@@ -216,7 +222,8 @@ namespace OpenFaceOffline
 
             if (record_HOG)
                 face_analyser.StopHOGRecording();
-            
+
+            face_analyser.PostProcessOutputFile(out_filename, dynamic_AU_model);
         }
 
     }

lib/local/CppInerop/FaceAnalyserInterop.h

@@ -260,6 +260,11 @@ public:
 		tracked_vid_writer->write(*tracked_face);
 	}
 
+	void PostProcessOutputFile(System::String^ file, bool dynamic)
+	{		
+		face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file), dynamic);
+	}
+
 	void AddNextFrame(OpenCVWrappers::RawImage^ frame, CppInterop::LandmarkDetector::CLNF^ clnf, double fx, double fy, double cx, double cy, bool online, bool vis_hog, bool vis_tracked) {
 			
 		face_analyser->AddNextFrame(frame->Mat, *clnf->getCLNF(), 0, online, vis_hog);

lib/local/FaceAnalyser/include/FaceAnalyser.h

@@ -126,6 +126,9 @@ public:
 	void ExtractAllPredictionsOfflineReg(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps, bool dynamic);
 	void ExtractAllPredictionsOfflineClass(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps, bool dynamic);
 
+	// Helper function for post-processing AU output files
+	void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic);
+
 private:
 
 	// Where the predictions are kept

lib/local/FaceAnalyser/src/FaceAnalyser.cpp

@@ -437,7 +437,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL
 	std::vector<std::pair<std::string, double>> AU_predictions_reg_corrected;
 	if(online)
 	{
-		AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success);
+		AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success, true);
 	}
 
 	// Add the reg predictions to the historic data
@@ -1255,4 +1255,122 @@ void FaceAnalyser::ReadRegressor(std::string fname, const vector<string>& au_nam
 
 double FaceAnalyser::GetCurrentTimeSeconds() {
 	return current_time_seconds;
-}
+}
+
+// Allows for post processing of the AU signal
+void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic)
+{
+
+	vector<double> certainties;
+	vector<bool> successes;
+	vector<double> timestamps;
+	vector<std::pair<std::string, vector<double>>> predictions_reg;
+	vector<std::pair<std::string, vector<double>>> predictions_class;
+
+	// Construct the new values to overwrite the output file with
+	ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic);
+	ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic);
+
+	int num_class = predictions_class.size();
+	int num_reg = predictions_reg.size();
+
+	// Extract the indices of writing out first
+	vector<string> au_reg_names = GetAURegNames();
+	std::sort(au_reg_names.begin(), au_reg_names.end());
+	vector<int> inds_reg;
+
+	// write out ar the correct index
+	for (string au_name : au_reg_names)
+	{
+		for (int i = 0; i < num_reg; ++i)
+		{
+			if (au_name.compare(predictions_reg[i].first) == 0)
+			{
+				inds_reg.push_back(i);
+				break;
+			}
+		}
+	}
+
+	vector<string> au_class_names = GetAUClassNames();
+	std::sort(au_class_names.begin(), au_class_names.end());
+	vector<int> inds_class;
+
+	// write out ar the correct index
+	for (string au_name : au_class_names)
+	{
+		for (int i = 0; i < num_class; ++i)
+		{
+			if (au_name.compare(predictions_class[i].first) == 0)
+			{
+				inds_class.push_back(i);
+				break;
+			}
+		}
+	}
+	// Read all of the output file in
+	vector<string> output_file_contents;
+
+	std::ifstream infile(output_file);
+	string line;
+
+	while (std::getline(infile, line))
+		output_file_contents.push_back(line);
+
+	infile.close();
+
+	// Read the header and find all _r and _c parts in a file and use their indices
+	std::vector<std::string> tokens;
+	boost::split(tokens, output_file_contents[0], boost::is_any_of(","));
+
+	int begin_ind = -1;
+
+	for (size_t i = 0; i < tokens.size(); ++i)
+	{
+		if (tokens[i].find("AU") != string::npos && begin_ind == -1)
+		{
+			begin_ind = i;
+			break;
+		}
+	}
+	int end_ind = begin_ind + num_class + num_reg;
+
+	// Now overwrite the whole file
+	std::ofstream outfile(output_file, ios_base::out);
+	// Write the header
+	outfile << std::setprecision(4);
+	outfile << output_file_contents[0].c_str() << endl;
+
+	// Write the contents
+	for (int i = 1; i < (int)output_file_contents.size(); ++i)
+	{
+		std::vector<std::string> tokens;
+		boost::split(tokens, output_file_contents[i], boost::is_any_of(","));
+
+		boost::trim(tokens[0]);
+		outfile << tokens[0];
+
+		for (int t = 1; t < (int)tokens.size(); ++t)
+		{
+			if (t >= begin_ind && t < end_ind)
+			{
+				if (t - begin_ind < num_reg)
+				{
+					outfile << ", " << predictions_reg[inds_reg[t - begin_ind]].second[i - 1];
+				}
+				else
+				{
+					outfile << ", " << predictions_class[inds_class[t - begin_ind - num_reg]].second[i - 1];
+				}
+			}
+			else
+			{
+				boost::trim(tokens[t]);
+				outfile << ", " << tokens[t];
+			}
+		}
+		outfile << endl;
+	}
+
+
+}