From f00d1fe51e201ed4f5a048308c51af6914b18180 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Thu, 2 Nov 2017 09:06:53 +0000 Subject: [PATCH] Working on the recorder interface. --- lib/local/Recorder/include/RecorderHOG.h | 5 +- lib/local/Recorder/include/RecorderOpenFace.h | 57 ++++++++++++++++--- lib/local/Recorder/src/RecorderCSV.cpp | 9 ++- lib/local/Recorder/src/RecorderHOG.cpp | 2 +- lib/local/Recorder/src/RecorderOpenFace.cpp | 50 ++++++++++++++-- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/lib/local/Recorder/include/RecorderHOG.h b/lib/local/Recorder/include/RecorderHOG.h index 4a21451..2e05902 100644 --- a/lib/local/Recorder/include/RecorderHOG.h +++ b/lib/local/Recorder/include/RecorderHOG.h @@ -38,6 +38,9 @@ #include #include +#include +#include + namespace Recorder { @@ -53,7 +56,7 @@ namespace Recorder RecorderHOG(); // Adding observations to the recorder - void AddObservationHOG(bool success, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); + void SetObservationHOG(bool success, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); bool Open(std::string filename); diff --git a/lib/local/Recorder/include/RecorderOpenFace.h b/lib/local/Recorder/include/RecorderOpenFace.h index befd083..f0b5414 100644 --- a/lib/local/Recorder/include/RecorderOpenFace.h +++ b/lib/local/Recorder/include/RecorderOpenFace.h @@ -67,17 +67,32 @@ namespace Recorder void Close(); // Adding observations to the recorder - void AddObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D); - void AddObservationLandmarkParameters(const cv::Vec6d& params_global, const cv::Mat_& params_local); - void AddObservationPose(const cv::Vec6d& pose); - void AddObservationActionUnits(const std::vector >& au_intensities, + + // Required observations for video/image-sequence + void SetObservationTimestamp(double timestamp); + + // All observations relevant to facial landmarks + void SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, + const cv::Vec6d& params_global, const cv::Mat_& params_local, double confidence, bool success); + + // Pose related observations + void SetObservationPose(const cv::Vec6d& pose); + + // AU related observations + void SetObservationActionUnits(const std::vector >& au_intensities, const std::vector >& au_occurences); - void AddObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, + + // Gaze related observations + void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const cv::Mat_& eye_landmarks); - void AddObservationFaceAlign(const cv::Mat& aligned_face); - void AddObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); - void AddObservationSuccess(double confidence, bool success); - void AddObservationTimestamp(double timestamp); + + // Face alignment related observations + void SetObservationFaceAlign(const cv::Mat& aligned_face); + + // HOG feature related observations + void SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); + + void WriteObservation(); private: @@ -103,6 +118,30 @@ namespace Recorder bool output_tracked_video; bool output_aligned_faces; + // The actual temporary storage for the observations + double timestamp; + + // Facial landmark related observations + cv::Mat_ landmarks_2D; + cv::Mat_ landmarks_3D; + cv::Vec6d params_global; + cv::Mat_ params_local; + double landmark_detection_confidence; + bool landmark_detection_success; + + // Head pose related observations + cv::Vec6d head_pose; + + // Action Unit related observations + std::vector > au_intensities; + std::vector > au_occurences; + + // Gaze related observations + cv::Point3f& gazeDirection0; + cv::Point3f& gazeDirection1; + cv::Vec2d& gaze_angle; + cv::Mat_& eye_landmarks; + }; } #endif \ No newline at end of file diff --git a/lib/local/Recorder/src/RecorderCSV.cpp b/lib/local/Recorder/src/RecorderCSV.cpp index 93d2b1a..b9d3a8a 100644 --- a/lib/local/Recorder/src/RecorderCSV.cpp +++ b/lib/local/Recorder/src/RecorderCSV.cpp @@ -44,11 +44,14 @@ RecorderCSV::RecorderCSV():output_file(){}; // TODO the other 4 constructors + destructors? // Opening the file and preparing the header for it -bool RecorderCSV::open(std::string output_file_name, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, +bool RecorderCSV::Open(std::string output_file_name, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, int num_face_landmarks, int num_model_modes, int num_eye_landmarks, std::vector au_names_class, std::vector au_names_reg) { output_file.open(output_file_name, std::ios_base::out); + if (!output_file.is_open()) + return false; + // Different headers if we are writing out the results on a sequence or an individual image if(this->is_sequence) { @@ -133,6 +136,8 @@ bool RecorderCSV::open(std::string output_file_name, bool output_2D_landmarks, b output_file << std::endl; + return true; + } // TODO check if the stream is open @@ -141,7 +146,7 @@ bool RecorderCSV::open(std::string output_file_name, bool output_2D_landmarks, b // const FaceAnalysis::FaceAnalyser& face_analyser); // Closing the file and cleaning up -void RecorderCSV::close() +void RecorderCSV::Close() { output_file.close(); } diff --git a/lib/local/Recorder/src/RecorderHOG.cpp b/lib/local/Recorder/src/RecorderHOG.cpp index 8d56800..08bb442 100644 --- a/lib/local/Recorder/src/RecorderHOG.cpp +++ b/lib/local/Recorder/src/RecorderHOG.cpp @@ -56,7 +56,7 @@ void RecorderHOG::Close() } // Writing to a HOG file -void RecorderHOG::AddObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels) +void RecorderHOG::SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels) { hog_file.write((char*)(&num_cols), 4); diff --git a/lib/local/Recorder/src/RecorderOpenFace.cpp b/lib/local/Recorder/src/RecorderOpenFace.cpp index 1ff51c7..e74633c 100644 --- a/lib/local/Recorder/src/RecorderOpenFace.cpp +++ b/lib/local/Recorder/src/RecorderOpenFace.cpp @@ -50,7 +50,7 @@ using namespace boost::filesystem; using namespace Recorder; -void create_directory(std::string output_path) +void CreateDirectory(std::string output_path) { // Creating the right directory structure @@ -81,11 +81,11 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s record_root = out_directory; // Construct the directories required for the output - create_directory(record_root); + CreateDirectory(record_root); // Create the required individual recorders, CSV, HOG, aligned, video std::string csv_filename = (path(record_root) / path(filename).replace_extension(".csv")).string(); - csv_recorder.open(csv_filename, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, + csv_recorder.Open(csv_filename, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg); // Consruct HOG recorder here @@ -101,10 +101,50 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s } -void RecorderOpenFace::AddObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels) +void RecorderOpenFace::SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels) { - hog_recorder.AddObservationHOG(good_frame, hog_descriptor, num_cols, num_rows, num_channels); + this->hog_recorder.SetObservationHOG(good_frame, hog_descriptor, num_cols, num_rows, num_channels); } +void RecorderOpenFace::SetObservationTimestamp(double timestamp) +{ + this->timestamp = timestamp; +} + +void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, + const cv::Vec6d& params_global, const cv::Mat_& params_local, double confidence, bool success) +{ + this->landmarks_2D = landmarks_2D; + this->landmarks_3D = landmarks_3D; + this->params_global = params_global; + this->params_local = params_local; + this->landmark_detection_confidence = confidence; + this->landmark_detection_success = success; + +} + +void RecorderOpenFace::SetObservationPose(const cv::Vec6d& pose) +{ + this->head_pose = pose; +} + +void RecorderOpenFace::SetObservationActionUnits(const std::vector >& au_intensities, + const std::vector >& au_occurences) +{ + this->au_intensities = au_intensities; + this->au_occurences = au_occurences; +} + +void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, + const cv::Vec2d& gaze_angle, const cv::Mat_& eye_landmarks) +{ + this->gazeDirection0 = gazeDirection0; + this->gazeDirection1 = gazeDirection1; + this->gaze_angle = gaze_angle; + this->eye_landmarks = eye_landmarks; +} + + + // TODO the other 4 constructors + destructors?