From b60669fa6214f52bea84d5f26c78cd1b7d61b994 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Fri, 3 Nov 2017 08:34:55 +0000 Subject: [PATCH] Continuing work on the recorder. --- exe/FeatureExtraction/FeatureExtraction.cpp | 123 +----------------- lib/local/Recorder/include/RecorderOpenFace.h | 24 +++- lib/local/Recorder/src/RecorderCSV.cpp | 5 +- lib/local/Recorder/src/RecorderOpenFace.cpp | 66 ++++++++-- 4 files changed, 84 insertions(+), 134 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index a2adf9b..5b40ed4 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -203,10 +203,6 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f } } -void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, - bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector au_names_class, vector au_names_reg); - // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow) void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, @@ -224,7 +220,8 @@ int main (int argc, char **argv) // Get the input output file parameters - string output_codec; //not used but should + string output_codec; + // TODO rename LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, output_codec, arguments); bool video_input = true; @@ -312,6 +309,7 @@ int main (int argc, char **argv) double fps_vid_in = -1.0; + // TODO this should be moved to a SequenceCapture class if(video_input) { // We might specify multiple video files as arguments @@ -391,27 +389,8 @@ int main (int argc, char **argv) // TODO this should always be video input int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); // TODO empty file check replaced Recorder::RecorderOpenFace openFaceRec(output_files[f_n], input_files[f_n], true, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, !output_hog_align_files.empty(), - !tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); - - // Creating output files - std::ofstream output_file; - - if (!output_files.empty()) - { - output_file.open(output_files[f_n], ios_base::out); - int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); + !tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames(), output_codec, fps_vid_in); - prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); - } - - - // Saving the HOG features - std::ofstream hog_output_file; - if(!output_hog_align_files.empty()) - { - hog_output_file.open(output_hog_align_files[f_n], ios_base::out | ios_base::binary); - } - // saving the videos cv::VideoWriter writerFace; if(!tracked_videos_output.empty()) @@ -424,19 +403,10 @@ int main (int argc, char **argv) { WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); } - - } int frame_count = 0; - - // This is useful for a second pass run (if want AU predictions) - // TODO remove these - vector params_global_video; - vector successes_video; - vector> params_local_video; - vector> detected_landmarks_video; - + // Use for timestamping if using a webcam int64 t_initial = cv::getTickCount(); @@ -649,89 +619,6 @@ int main (int argc, char **argv) return 0; } -void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, - bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector au_names_class, vector au_names_reg) -{ - - *output_file << "frame, timestamp, confidence, success"; - - if (output_gaze) - { - *output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y"; - - for (int i = 0; i < num_eye_landmarks; ++i) - { - *output_file << ", eye_lmk_x_" << i; - } - for (int i = 0; i < num_eye_landmarks; ++i) - { - *output_file << ", eye_lmk_y_" << i; - } - } - - if (output_pose) - { - *output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz"; - } - - if (output_2D_landmarks) - { - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", x_" << i; - } - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", y_" << i; - } - } - - if (output_3D_landmarks) - { - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", X_" << i; - } - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", Y_" << i; - } - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", Z_" << i; - } - } - - // Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters - if (output_model_params) - { - *output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty"; - for (int i = 0; i < num_model_modes; ++i) - { - *output_file << ", p_" << i; - } - } - - if (output_AUs) - { - std::sort(au_names_reg.begin(), au_names_reg.end()); - for (string reg_name : au_names_reg) - { - *output_file << ", " << reg_name << "_r"; - } - - std::sort(au_names_class.begin(), au_names_class.end()); - for (string class_name : au_names_class) - { - *output_file << ", " << class_name << "_c"; - } - } - - *output_file << endl; - -} - // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow) void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, diff --git a/lib/local/Recorder/include/RecorderOpenFace.h b/lib/local/Recorder/include/RecorderOpenFace.h index ad747d5..504983a 100644 --- a/lib/local/Recorder/include/RecorderOpenFace.h +++ b/lib/local/Recorder/include/RecorderOpenFace.h @@ -39,7 +39,10 @@ // System includes #include + +// OpenCV includes #include +#include namespace Recorder { @@ -55,9 +58,9 @@ namespace Recorder // The constructor for the recorder, need to specify if we are recording a sequence or not RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks, - const std::vector& au_names_class, const std::vector& au_names_reg); + const std::vector& au_names_class, const std::vector& au_names_reg, const std::string& output_codec, double fps_vid_in); - // Simplified constructor that records all + // Simplified constructor that records all, TODO implement RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector& au_names_class, const std::vector& au_names_reg); @@ -92,6 +95,8 @@ namespace Recorder // HOG feature related observations void SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); + void SetObservationVisualization(const cv::Mat_ &vis_track); + void WriteObservation(); private: @@ -124,8 +129,8 @@ namespace Recorder // Facial landmark related observations cv::Mat_ landmarks_2D; cv::Mat_ landmarks_3D; - cv::Vec6d params_global; - cv::Mat_ params_local; + cv::Vec6d pdm_params_global; + cv::Mat_ pdm_params_local; double landmark_detection_confidence; bool landmark_detection_success; @@ -137,13 +142,20 @@ namespace Recorder std::vector > au_occurences; // Gaze related observations - cv::Point3f gazeDirection0; - cv::Point3f gazeDirection1; + cv::Point3f gaze_direction0; + cv::Point3f gaze_direction1; cv::Vec2d gaze_angle; cv::Mat_ eye_landmarks; int observation_count; + // For video writing + cv::VideoWriter video_writer; + std::string video_filename; + std::string output_codec; + double fps_vid_out; + cv::Mat_ vis_to_out; + }; } #endif \ No newline at end of file diff --git a/lib/local/Recorder/src/RecorderCSV.cpp b/lib/local/Recorder/src/RecorderCSV.cpp index 278d484..fea7241 100644 --- a/lib/local/Recorder/src/RecorderCSV.cpp +++ b/lib/local/Recorder/src/RecorderCSV.cpp @@ -36,6 +36,9 @@ // For sorting #include +// For standard out +#include + using namespace Recorder; // Default constructor initializes the variables @@ -45,7 +48,7 @@ RecorderCSV::RecorderCSV():output_file(){}; // Opening the file and preparing the header for it bool RecorderCSV::Open(std::string output_file_name, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector au_names_class, const std::vector au_names_reg) + int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector& au_names_class, const std::vector& au_names_reg) { output_file.open(output_file_name, std::ios_base::out); diff --git a/lib/local/Recorder/src/RecorderOpenFace.cpp b/lib/local/Recorder/src/RecorderOpenFace.cpp index 720f4ad..124e13a 100644 --- a/lib/local/Recorder/src/RecorderOpenFace.cpp +++ b/lib/local/Recorder/src/RecorderOpenFace.cpp @@ -50,6 +50,9 @@ using namespace boost::filesystem; using namespace Recorder; +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + void CreateDirectory(std::string output_path) { @@ -70,10 +73,10 @@ void CreateDirectory(std::string output_path) RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks, - const std::vector& au_names_class, const std::vector& au_names_reg): + const std::vector& au_names_class, const std::vector& au_names_reg, const std::string& output_codec, double fps_vid_in): is_sequence(sequence), output_2D_landmarks(output_2D_landmarks), output_3D_landmarks(output_3D_landmarks), output_aligned_faces(output_aligned_faces), output_AUs(output_AUs), output_gaze(output_gaze), output_hog(output_hog), output_model_params(output_model_params), - output_pose(output_pose), output_tracked_video(output_tracked_video) + output_pose(output_pose), output_tracked_video(output_tracked_video), video_writer(), fps_vid_out(fps_vid_in), output_codec(output_codec) { // From the filename, strip out the name without directory and extension @@ -96,6 +99,13 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s } // TODO construct a video recorder + // saving the videos + + if (output_tracked_video) + { + this->video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string(); + } + // Prepare image recording @@ -103,14 +113,52 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s } +void RecorderOpenFace::SetObservationVisualization(const cv::Mat_ &vis_track) +{ + if (output_tracked_video) + { + // Initialize the video writer if it has not been opened yet + if(!video_writer.isOpened()) + { + std::string video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string(); + try + { + video_writer.open(video_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), fps_vid_out, vis_track.size(), true); + } + catch (cv::Exception e) + { + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); + } + } + + vis_to_out = vis_track; + + } + +} + + void RecorderOpenFace::WriteObservation() { observation_count++; // Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers) - this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success; + this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, + landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose, + gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks, au_intensities, au_occurences); // TODO HOG + + if(output_tracked_video) + { + if (vis_to_out.empty) + { + WARN_STREAM("Output tracked video frame is not set"); + } + video_writer.write(vis_to_out); + // Clear the output + vis_to_out = cv::Mat(); + } } @@ -125,12 +173,12 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp) } void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, - const cv::Vec6d& params_global, const cv::Mat_& params_local, double confidence, bool success) + const cv::Vec6d& pdm_params_global, const cv::Mat_& pdm_params_local, double confidence, bool success) { this->landmarks_2D = landmarks_2D; this->landmarks_3D = landmarks_3D; - this->params_global = params_global; - this->params_local = params_local; + this->pdm_params_global = pdm_params_global; + this->pdm_params_local = pdm_params_local; this->landmark_detection_confidence = confidence; this->landmark_detection_success = success; @@ -148,11 +196,11 @@ void RecorderOpenFace::SetObservationActionUnits(const std::vectorau_occurences = au_occurences; } -void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, +void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const cv::Vec2d& gaze_angle, const cv::Mat_& eye_landmarks) { - this->gazeDirection0 = gazeDirection0; - this->gazeDirection1 = gazeDirection1; + this->gaze_direction0 = gaze_direction0; + this->gaze_direction1 = gaze_direction1; this->gaze_angle = gaze_angle; this->eye_landmarks = eye_landmarks; }