Interface changes to provide multi output support, also enforcing parameters in executables.

exe/FaceLandmarkImg/FaceLandmarkImg.cpp

@@ -85,6 +85,14 @@ int main (int argc, char **argv)
 	//Convert arguments to more convenient vector form
 	vector<string> arguments = get_arguments(argc, argv);
 
+	// no arguments: output usage
+	if (arguments.size() == 1)
+	{
+		cout << "For command line arguments see:" << endl;
+		cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
+		return 0;
+	}
+
 	// Prepare for image reading
 	Utilities::ImageCapture image_reader;
 
@@ -205,6 +213,7 @@ int main (int argc, char **argv)
 			open_face_rec.SetObservationPose(pose_estimate);
 			open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy));
 			open_face_rec.SetObservationFaceAlign(sim_warped_img);
+			open_face_rec.SetObservationFaceID(face);
 			open_face_rec.WriteObservation();
 
 		}

exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp

@@ -39,6 +39,10 @@
 #include "VisualizationUtils.h"
 #include "Visualizer.h"
 #include "SequenceCapture.h"
+#include <RecorderOpenFace.h>
+#include <RecorderOpenFaceParameters.h>
+#include <GazeEstimation.h>
+#include <FaceAnalyser.h>
 
 #include <fstream>
 #include <sstream>
@@ -110,7 +114,14 @@ int main (int argc, char **argv)
 
 	vector<string> arguments = get_arguments(argc, argv);
 
-			
+	// no arguments: output usage
+	if (arguments.size() == 1)
+	{
+		cout << "For command line arguments see:" << endl;
+		cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
+		return 0;
+	}
+
 	LandmarkDetector::FaceModelParameters det_params(arguments);
 	// This is so that the model would not try re-initialising itself
 	det_params.reinit_video_every = -1;
@@ -142,11 +153,16 @@ int main (int argc, char **argv)
 		det_parameters.push_back(det_params);
 	}
 	
+	// Load facial feature extractor and AU analyser (make sure it is static, as we don't reidentify faces)
+	FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
+	face_analysis_params.OptimizeForImages();
+	FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
+
 	// Open a sequence
 	Utilities::SequenceCapture sequence_reader;
 
 	// A utility for visualizing the results (show just the tracks)
-	Utilities::Visualizer visualizer(true, false, false);
+	Utilities::Visualizer visualizer(arguments);
 
 	// Tracking FPS for visualization
 	Utilities::FpsTracker fps_tracker;
@@ -159,46 +175,41 @@ int main (int argc, char **argv)
 
 		// The sequence reader chooses what to open based on command line arguments provided
 		if (!sequence_reader.Open(arguments))
-		{
+			break;
-			// If failed to open because no input files specified, attempt to open a webcam
+
-			if (sequence_reader.no_input_specified && sequence_number == 0)
-			{
-				// If that fails, revert to webcam
-				INFO_STREAM("No input specified, attempting to open a webcam 0");
-				if (!sequence_reader.OpenWebcam(0))
-				{
-					ERROR_STREAM("Failed to open the webcam");
-					break;
-				}
-			}
-			else
-			{
-				break;
-			}
-		}
 		INFO_STREAM("Device or file opened");
 
 		cv::Mat captured_image = sequence_reader.GetNextFrame();
 
 		int frame_count = 0;
 
+		Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(),
+			sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps);
+		// Do not do AU detection on multi-face case as it is not supported
+		recording_params.setOutputAUs(false);
+		Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
+
+		if (recording_params.outputGaze() && !face_model.eye_model)
+			cout << "WARNING: no eye model defined, but outputting gaze" << endl;
+
+		if (sequence_reader.IsWebcam())
+		{
+			INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)");
+			visualizer.vis_track = true;
+		}
+
+		if (recording_params.outputAUs())
+		{
+			INFO_STREAM("WARNING: using a AU detection in multiple face mode, it might not be as accurate and is experimental");
+		}
+
+
 		INFO_STREAM( "Starting tracking");
 		while(!captured_image.empty())
 		{		
 
 			// Reading the images
-			cv::Mat_<uchar> grayscale_image;
+			cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
-
-			cv::Mat disp_image = captured_image.clone();
-
-			if(captured_image.channels() == 3)
-			{
-				cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY);				
-			}
-			else
-			{
-				grayscale_image = captured_image.clone();				
-			}
 		
 			vector<cv::Rect_<double> > face_detections;
 
@@ -261,7 +272,7 @@ int main (int argc, char **argv)
 							// This ensures that a wider window is used for the initial landmark localisation
 							face_models[model].detection_success = false;
 							detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], face_models[model], det_parameters[model]);
-													
+
 							// This activates the model
 							active_models[model] = true;
 
@@ -283,14 +294,60 @@ int main (int argc, char **argv)
 
 			visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
 
-			// Go through every model and visualise the results
+			// Go through every model and detect eye gaze, record results and visualise the results
 			for(size_t model = 0; model < face_models.size(); ++model)
 			{
 				// Visualising the results
 				if(active_models[model])
 				{
+
+					// Estimate head pose and eye gaze				
+					cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
+
+					cv::Point3f gaze_direction0(0, 0, 0); cv::Point3f gaze_direction1(0, 0, 0); cv::Vec2d gaze_angle(0, 0);
+
+					// Detect eye gazes
+					if (face_models[model].detection_success && face_model.eye_model)
+					{
+						GazeAnalysis::EstimateGaze(face_models[model], gaze_direction0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
+						GazeAnalysis::EstimateGaze(face_models[model], gaze_direction1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
+						gaze_angle = GazeAnalysis::GetGazeAngle(gaze_direction0, gaze_direction1);
+					}
+
+					// Face analysis step
+					cv::Mat sim_warped_img;
+					cv::Mat_<double> hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0;
+
+					// Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization
+					if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog)
+					{
+						face_analyser.PredictStaticAUsAndComputeFeatures(captured_image, face_models[model].detected_landmarks);
+						face_analyser.GetLatestAlignedFace(sim_warped_img);
+						face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols);
+					}
+
+					// Visualize the features
+					visualizer.SetObservationFaceAlign(sim_warped_img);
+					visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
 					visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty);
 					visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty);
+					visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty);
+
+					// Output features
+					open_face_rec.SetObservationHOG(face_models[model].detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG
+					open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
+					open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
+					open_face_rec.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
+						face_models[model].params_global, face_models[model].params_local, face_models[model].detection_certainty, face_models[model].detection_success);
+					open_face_rec.SetObservationPose(pose_estimate);
+					open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
+					open_face_rec.SetObservationFaceAlign(sim_warped_img);
+					open_face_rec.SetObservationFaceID(model);
+					open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
+					open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber());
+					open_face_rec.WriteObservation();
+
+
 				}
 			}
 			visualizer.SetFps(fps_tracker.GetFPS());

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -103,6 +103,14 @@ int main (int argc, char **argv)
 
 	vector<string> arguments = get_arguments(argc, argv);
 
+	// no arguments: output usage
+	if (arguments.size() == 1)
+	{
+		cout << "For command line arguments see:" << endl;
+		cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
+		return 0;
+	}
+
 	// Load the modules that are being used for tracking and face analysis
 	// Load face landmark detector
 	LandmarkDetector::FaceModelParameters det_parameters(arguments);
@@ -217,6 +225,8 @@ int main (int argc, char **argv)
 			open_face_rec.SetObservationPose(pose_estimate);
 			open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
 			open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
+			open_face_rec.SetObservationFaceID(0);
+			open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber());
 			open_face_rec.SetObservationFaceAlign(sim_warped_img);
 			open_face_rec.WriteObservation();
 			

lib/local/Utilities/include/RecorderCSV.h

@@ -60,10 +60,12 @@ namespace Utilities
 		bool Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
 			int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
 
+		bool isOpen() const { return output_file.is_open(); }
+
 		// Closing the file and cleaning up
 		void Close();
 
-		void WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence,
+		void WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
 			const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D, const cv::Mat_<double>& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate,
 			const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d,
 			const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);

lib/local/Utilities/include/RecorderOpenFace.h

@@ -70,6 +70,12 @@ namespace Utilities
 		// Required observations for video/image-sequence
 		void SetObservationTimestamp(double timestamp);
 
+		// Required observations for video/image-sequence
+		void SetObservationFrameNumber(double frame_number);
+
+		// If in multiple face mode, identifying which face was tracked
+		void SetObservationFaceID(int face_id);
+
 		// All observations relevant to facial landmarks
 		void SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D, 
 			const cv::Vec6d& params_global, const cv::Mat_<double>& params_local, double confidence, bool success);
@@ -123,7 +129,10 @@ namespace Utilities
 		RecorderHOG hog_recorder;
 
 		// The actual temporary storage for the observations
+		
 		double timestamp;
+		int face_id;
+		int frame_number;
 
 		// Facial landmark related observations
 		cv::Mat_<double> landmarks_2D;
@@ -147,7 +156,8 @@ namespace Utilities
 		std::vector<cv::Point2d> eye_landmarks2D;
 		std::vector<cv::Point3d> eye_landmarks3D;
 
-		int observation_count;
+		int m_frame_number; // TODO this should be set
+		int m_face_id; // TODO this should be set
 
 		// For video writing
 		cv::VideoWriter video_writer;

lib/local/Utilities/include/RecorderOpenFaceParameters.h

@@ -77,6 +77,8 @@ namespace Utilities
 		float getCx() const { return cx; }
 		float getCy() const { return cy; }
 
+		void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; }
+
 	private:
 		
 		// If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face

lib/local/Utilities/include/SequenceCapture.h

@@ -85,6 +85,8 @@ namespace Utilities
 		// Parameters describing the sequence and it's progress
 		double GetProgress();
 
+		int GetFrameNumber() { return frame_num; }
+
 		bool IsOpened();
 
 		void Close();

lib/local/Utilities/src/RecorderCSV.cpp

@@ -78,7 +78,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp
 	// Different headers if we are writing out the results on a sequence or an individual image
 	if(this->is_sequence)
 	{
-		output_file << "frame, timestamp, confidence, success";
+		output_file << "frame, face_id, timestamp, confidence, success";
 	}
 	else
 	{
@@ -176,7 +176,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp
 
 }
 
-void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence,
+void RecorderCSV::WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
 	const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D, const cv::Mat_<double>& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate,
 	const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d,
 	const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences)
@@ -193,8 +193,9 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm
 	output_file << std::noshowpoint;
 	if(is_sequence)
 	{
+		
 		output_file << std::setprecision(3);
-		output_file << observation_count << ", " << time_stamp;
+		output_file << frame_num << ", " << face_id << ", " << time_stamp;
 		output_file << std::setprecision(2);
 		output_file << ", " << landmark_confidence;
 		output_file << std::setprecision(0);
@@ -203,7 +204,7 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm
 	else
 	{
 		output_file << std::setprecision(3);
-		output_file << observation_count << ", " << landmark_confidence;
+		output_file << face_id << ", " << landmark_confidence;
 	}
 	// Output the estimated gaze
 	if (output_gaze)

lib/local/Utilities/src/RecorderOpenFace.cpp

@@ -153,7 +153,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename)
 		CreateDirectory(aligned_output_directory);
 	}
 
-	observation_count = 0;
+	this->frame_number = 0;
 
 }
 
@@ -275,11 +275,9 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track)
 
 void RecorderOpenFace::WriteObservation()
 {
-	observation_count++;
 
-	// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
+	// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)	
-	
+	if(!csv_recorder.isOpen())
-	if(observation_count == 1)
 	{
 		// As we are writing out the header, work out some things like number of landmarks, names of AUs etc.
 		int num_face_landmarks = landmarks_2D.rows / 2;
@@ -315,7 +313,7 @@ void RecorderOpenFace::WriteObservation()
 			params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg);
 	}
 
-	this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, 
+	this->csv_recorder.WriteLine(face_id, frame_number, timestamp, landmark_detection_success, 
 		landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
 		gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences);
 
@@ -331,9 +329,9 @@ void RecorderOpenFace::WriteObservation()
 
 		// Filename is based on frame number
 		if(params.isSequence())
-			std::sprintf(name, "frame_det_%06d.bmp", observation_count);
+			std::sprintf(name, "frame_det_%02d_%06d.bmp", face_id, frame_number);
 		else
-			std::sprintf(name, "face_det_%06d.bmp", observation_count);
+			std::sprintf(name, "face_det_%06d.bmp", face_id);
 
 		// Construct the output filename
 		boost::filesystem::path slash("/");
@@ -387,6 +385,19 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp)
 	this->timestamp = timestamp;
 }
 
+// Required observations for video/image-sequence
+void RecorderOpenFace::SetObservationFrameNumber(double frame_number)
+{
+	this->frame_number = frame_number;
+}
+
+// If in multiple face mode, identifying which face was tracked
+void RecorderOpenFace::SetObservationFaceID(int face_id)
+{
+	this->face_id = face_id;
+}
+
+
 void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D,
 	const cv::Vec6d& pdm_params_global, const cv::Mat_<double>& pdm_params_local, double confidence, bool success)
 {

lib/local/Utilities/src/RecorderOpenFaceParameters.cpp

@@ -60,61 +60,61 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector<std::string>
 
 	bool output_set = false;
 
-	output_2D_landmarks = false;
+	this->output_2D_landmarks = false;
-	output_3D_landmarks = false;
+	this->output_3D_landmarks = false;
-	output_model_params = false;
+	this->output_model_params = false;
-	output_pose = false;
+	this->output_pose = false;
-	output_AUs = false;
+	this->output_AUs = false;
-	output_gaze = false;
+	this->output_gaze = false;
-	output_hog = false;
+	this->output_hog = false;
-	output_tracked = false;
+	this->output_tracked = false;
-	output_aligned_faces = false;
+	this->output_aligned_faces = false;
 
 	for (size_t i = 0; i < arguments.size(); ++i)
 	{
 		if (arguments[i].compare("-simalign") == 0)
 		{
-			output_aligned_faces = true;
+			this->output_aligned_faces = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-hogalign") == 0)
 		{
-			output_hog = true;
+			this->output_hog = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-2Dfp") == 0)
 		{
-			output_2D_landmarks = true;
+			this->output_2D_landmarks = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-3Dfp") == 0)
 		{
-			output_3D_landmarks = true;
+			this->output_3D_landmarks = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-pdmparams") == 0)
 		{
-			output_model_params = true;
+			this->output_model_params = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-pose") == 0)
 		{
-			output_pose = true;
+			this->output_pose = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-aus") == 0)
 		{
-			output_AUs = true;
+			this->output_AUs = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-gaze") == 0)
 		{
-			output_gaze = true;
+			this->output_gaze = true;
 			output_set = true;
 		}
 		else if (arguments[i].compare("-tracked") == 0)
 		{
-			output_tracked = true;
+			this->output_tracked = true;
 			output_set = true;
 		}
 	}
@@ -123,15 +123,15 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector<std::string>
 
 	if (!output_set)
 	{
-		output_2D_landmarks = true;
+		this->output_2D_landmarks = true;
-		output_3D_landmarks = true;
+		this->output_3D_landmarks = true;
-		output_model_params = true;
+		this->output_model_params = true;
-		output_pose = true;
+		this->output_pose = true;
-		output_AUs = true;
+		this->output_AUs = true;
-		output_gaze = true;
+		this->output_gaze = true;
-		output_hog = true;
+		this->output_hog = true;
-		output_tracked = true;
+		this->output_tracked = true;
-		output_aligned_faces = true;
+		this->output_aligned_faces = true;
 	}
 
 }

matlab_runners/Demos/run_demo_video_multi.m

@@ -19,7 +19,7 @@ model = 'model/main_clnf_general.txt'; % Trained on in the wild and multi-pie da
 %model = 'model/main_clm_wild.txt'; % Trained on in-the-wild
 
 % Create a command that will run the tracker on set of videos and display the output 
-command = sprintf('%s -mloc "%s" ', executable, model);
+command = sprintf('%s -mloc "%s" -verbose ', executable, model);
 
 % add all videos to single argument list (so as not to load the model anew
 % for every video)