Continuing work on the recorder.

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -203,10 +203,6 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
 	}
 }
 
-void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
-	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
-	int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg);
-
 // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
 void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
@@ -224,7 +220,8 @@ int main (int argc, char **argv)
 	
 	// Get the input output file parameters
 	
-	string output_codec; //not used but should
+	string output_codec;
+	// TODO rename
 	LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, output_codec, arguments);
 
 	bool video_input = true;
@@ -312,6 +309,7 @@ int main (int argc, char **argv)
 
 		double fps_vid_in = -1.0;
 
+		// TODO this should be moved to a SequenceCapture class
 		if(video_input)
 		{
 			// We might specify multiple video files as arguments
@@ -391,27 +389,8 @@ int main (int argc, char **argv)
 		// TODO this should always be video input
 		int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); // TODO empty file check replaced
 		Recorder::RecorderOpenFace openFaceRec(output_files[f_n], input_files[f_n], true, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, !output_hog_align_files.empty(),
-			!tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
+			!tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames(), output_codec, fps_vid_in);
-
-		// Creating output files
-		std::ofstream output_file;
-
-		if (!output_files.empty())
-		{
-			output_file.open(output_files[f_n], ios_base::out);
-			int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size();
 		
-			prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
-		}
-
-
-		// Saving the HOG features
-		std::ofstream hog_output_file;
-		if(!output_hog_align_files.empty())
-		{
-			hog_output_file.open(output_hog_align_files[f_n], ios_base::out | ios_base::binary);
-		}
-
 		// saving the videos
 		cv::VideoWriter writerFace;
 		if(!tracked_videos_output.empty())
@@ -424,19 +403,10 @@ int main (int argc, char **argv)
 			{
 				WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
 			}
-
-			
 		}
 
 		int frame_count = 0;
-		
+						
-		// This is useful for a second pass run (if want AU predictions)
-		// TODO remove these
-		vector<cv::Vec6d> params_global_video;
-		vector<bool> successes_video;
-		vector<cv::Mat_<double>> params_local_video;
-		vector<cv::Mat_<double>> detected_landmarks_video;
-				
 		// Use for timestamping if using a webcam
 		int64 t_initial = cv::getTickCount();
 
@@ -649,89 +619,6 @@ int main (int argc, char **argv)
 	return 0;
 }
 
-void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
-	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
-	int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg)
-{
-
-	*output_file << "frame, timestamp, confidence, success";
-
-	if (output_gaze)
-	{
-		*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
-
-		for (int i = 0; i < num_eye_landmarks; ++i)
-		{
-			*output_file << ", eye_lmk_x_" << i;
-		}
-		for (int i = 0; i < num_eye_landmarks; ++i)
-		{
-			*output_file << ", eye_lmk_y_" << i;
-		}
-	}
-
-	if (output_pose)
-	{
-		*output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz";
-	}
-
-	if (output_2D_landmarks)
-	{
-		for (int i = 0; i < num_face_landmarks; ++i)
-		{
-			*output_file << ", x_" << i;
-		}
-		for (int i = 0; i < num_face_landmarks; ++i)
-		{
-			*output_file << ", y_" << i;
-		}
-	}
-
-	if (output_3D_landmarks)
-	{
-		for (int i = 0; i < num_face_landmarks; ++i)
-		{
-			*output_file << ", X_" << i;
-		}
-		for (int i = 0; i < num_face_landmarks; ++i)
-		{
-			*output_file << ", Y_" << i;
-		}
-		for (int i = 0; i < num_face_landmarks; ++i)
-		{
-			*output_file << ", Z_" << i;
-		}
-	}
-
-	// Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters
-	if (output_model_params)
-	{
-		*output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty";
-		for (int i = 0; i < num_model_modes; ++i)
-		{
-			*output_file << ", p_" << i;
-		}
-	}
-
-	if (output_AUs)
-	{
-		std::sort(au_names_reg.begin(), au_names_reg.end());
-		for (string reg_name : au_names_reg)
-		{
-			*output_file << ", " << reg_name << "_r";
-		}
-
-		std::sort(au_names_class.begin(), au_names_class.end());
-		for (string class_name : au_names_class)
-		{
-			*output_file << ", " << class_name << "_c";
-		}
-	}
-
-	*output_file << endl;
-
-}
-
 // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
 void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,

lib/local/Recorder/include/RecorderOpenFace.h

@@ -39,7 +39,10 @@
 
 // System includes
 #include <vector>
+
+// OpenCV includes
 #include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
 
 namespace Recorder
 {
@@ -55,9 +58,9 @@ namespace Recorder
 		// The constructor for the recorder, need to specify if we are recording a sequence or not
 		RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose,
 			bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks, 
-			const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
+			const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg, const std::string& output_codec, double fps_vid_in);
 
-		// Simplified constructor that records all
+		// Simplified constructor that records all, TODO implement
 		RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
 			const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
 
@@ -92,6 +95,8 @@ namespace Recorder
 		// HOG feature related observations
 		void SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
 
+		void SetObservationVisualization(const cv::Mat_<double> &vis_track);
+
 		void WriteObservation();
 
 	private:
@@ -124,8 +129,8 @@ namespace Recorder
 		// Facial landmark related observations
 		cv::Mat_<double> landmarks_2D;
 		cv::Mat_<double> landmarks_3D;
-		cv::Vec6d params_global;
+		cv::Vec6d pdm_params_global;
-		cv::Mat_<double> params_local;
+		cv::Mat_<double> pdm_params_local;
 		double landmark_detection_confidence;
 		bool landmark_detection_success;
 
@@ -137,13 +142,20 @@ namespace Recorder
 		std::vector<std::pair<std::string, double> > au_occurences;
 
 		// Gaze related observations
-		cv::Point3f gazeDirection0;
+		cv::Point3f gaze_direction0;
-		cv::Point3f gazeDirection1;
+		cv::Point3f gaze_direction1;
 		cv::Vec2d gaze_angle;
 		cv::Mat_<double> eye_landmarks;
 
 		int observation_count;
 
+		// For video writing
+		cv::VideoWriter video_writer;
+		std::string video_filename;
+		std::string output_codec;
+		double fps_vid_out;
+		cv::Mat_<double> vis_to_out;
+
 	};
 }
 #endif

lib/local/Recorder/src/RecorderCSV.cpp

@@ -36,6 +36,9 @@
 // For sorting
 #include <algorithm>
 
+// For standard out
+#include <iostream>
+
 using namespace Recorder;
 
 // Default constructor initializes the variables
@@ -45,7 +48,7 @@ RecorderCSV::RecorderCSV():output_file(){};
 
 // Opening the file and preparing the header for it
 bool RecorderCSV::Open(std::string output_file_name, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
-	int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string> au_names_class, const std::vector<std::string> au_names_reg)
+	int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg)
 {
 	output_file.open(output_file_name, std::ios_base::out);
 

lib/local/Recorder/src/RecorderOpenFace.cpp

@@ -50,6 +50,9 @@ using namespace boost::filesystem;
 
 using namespace Recorder;
 
+#define WARN_STREAM( stream ) \
+std::cout << "Warning: " << stream << std::endl
+
 void CreateDirectory(std::string output_path)
 {
 
@@ -70,10 +73,10 @@ void CreateDirectory(std::string output_path)
 
 RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose,
 	bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
-	const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg):
+	const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg, const std::string& output_codec, double fps_vid_in):
 	is_sequence(sequence), output_2D_landmarks(output_2D_landmarks), output_3D_landmarks(output_3D_landmarks), output_aligned_faces(output_aligned_faces),
 	output_AUs(output_AUs), output_gaze(output_gaze), output_hog(output_hog), output_model_params(output_model_params),
-	output_pose(output_pose), output_tracked_video(output_tracked_video)
+	output_pose(output_pose), output_tracked_video(output_tracked_video), video_writer(), fps_vid_out(fps_vid_in), output_codec(output_codec)
 {
 
 	// From the filename, strip out the name without directory and extension
@@ -96,6 +99,13 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s
 	}
 
 	// TODO construct a video recorder
+	// saving the videos
+	
+	if (output_tracked_video)
+	{
+		this->video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string();
+	}
+
 
 	// Prepare image recording
 
@@ -103,14 +113,52 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s
 
 }
 
+void RecorderOpenFace::SetObservationVisualization(const cv::Mat_<double> &vis_track)
+{
+	if (output_tracked_video)
+	{
+		// Initialize the video writer if it has not been opened yet
+		if(!video_writer.isOpened())
+		{
+			std::string video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string();
+			try
+			{
+				video_writer.open(video_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), fps_vid_out, vis_track.size(), true);
+			}
+			catch (cv::Exception e)
+			{
+				WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
+			}
+		}
+
+		vis_to_out = vis_track;
+
+	}
+
+}
+
+
 void RecorderOpenFace::WriteObservation()
 {
 	observation_count++;
 
 	// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
-	this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success;
+	this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, 
+		landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
+		gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks, au_intensities, au_occurences);
 
 	// TODO HOG
+
+	if(output_tracked_video)
+	{
+		if (vis_to_out.empty)
+		{
+			WARN_STREAM("Output tracked video frame is not set");
+		}
+		video_writer.write(vis_to_out);
+		// Clear the output
+		vis_to_out = cv::Mat();
+	}
 }
 
 
@@ -125,12 +173,12 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp)
 }
 
 void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D,
-	const cv::Vec6d& params_global, const cv::Mat_<double>& params_local, double confidence, bool success)
+	const cv::Vec6d& pdm_params_global, const cv::Mat_<double>& pdm_params_local, double confidence, bool success)
 {
 	this->landmarks_2D = landmarks_2D;
 	this->landmarks_3D = landmarks_3D;
-	this->params_global = params_global;
+	this->pdm_params_global = pdm_params_global;
-	this->params_local = params_local;
+	this->pdm_params_local = pdm_params_local;
 	this->landmark_detection_confidence = confidence;
 	this->landmark_detection_success = success;
 
@@ -148,11 +196,11 @@ void RecorderOpenFace::SetObservationActionUnits(const std::vector<std::pair<std
 	this->au_occurences = au_occurences;
 }
 
-void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1,
+void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1,
 	const cv::Vec2d& gaze_angle, const cv::Mat_<double>& eye_landmarks)
 {
-	this->gazeDirection0 = gazeDirection0;
+	this->gaze_direction0 = gaze_direction0;
-	this->gazeDirection1 = gazeDirection1;
+	this->gaze_direction1 = gaze_direction1;
 	this->gaze_angle = gaze_angle;
 	this->eye_landmarks = eye_landmarks;
 }