r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Continuing work on the recorder.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-11-03 08:34:55 +00:00
parent 4f4bf263d6
commit b60669fa62
4 changed files with 84 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
exe/FeatureExtraction/FeatureExtraction.cpp
View file

@ -203,10 +203,6 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
}
}
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg);
// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
@ -224,7 +220,8 @@ int main (int argc, char **argv)
// Get the input output file parameters
string output_codec; //not used but should
string output_codec;
// TODO rename
LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, output_codec, arguments);
bool video_input = true;
@ -312,6 +309,7 @@ int main (int argc, char **argv)
double fps_vid_in = -1.0;
// TODO this should be moved to a SequenceCapture class
if(video_input)
{
// We might specify multiple video files as arguments
@ -391,26 +389,7 @@ int main (int argc, char **argv)
// TODO this should always be video input
int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); // TODO empty file check replaced
Recorder::RecorderOpenFace openFaceRec(output_files[f_n], input_files[f_n], true, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, !output_hog_align_files.empty(),
!tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
// Creating output files
std::ofstream output_file;
if (!output_files.empty())
{
output_file.open(output_files[f_n], ios_base::out);
int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size();
prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
}
// Saving the HOG features
std::ofstream hog_output_file;
if(!output_hog_align_files.empty())
{
hog_output_file.open(output_hog_align_files[f_n], ios_base::out | ios_base::binary);
}
!tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames(), output_codec, fps_vid_in);
// saving the videos
cv::VideoWriter writerFace;
@ -424,19 +403,10 @@ int main (int argc, char **argv)
{
WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
}
}
int frame_count = 0;
// This is useful for a second pass run (if want AU predictions)
// TODO remove these
vector<cv::Vec6d> params_global_video;
vector<bool> successes_video;
vector<cv::Mat_<double>> params_local_video;
vector<cv::Mat_<double>> detected_landmarks_video;
// Use for timestamping if using a webcam
int64 t_initial = cv::getTickCount();
@ -649,89 +619,6 @@ int main (int argc, char **argv)
return 0;
}
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg)
{
*output_file << "frame, timestamp, confidence, success";
if (output_gaze)
{
*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
for (int i = 0; i < num_eye_landmarks; ++i)
{
*output_file << ", eye_lmk_x_" << i;
}
for (int i = 0; i < num_eye_landmarks; ++i)
{
*output_file << ", eye_lmk_y_" << i;
}
}
if (output_pose)
{
*output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz";
}
if (output_2D_landmarks)
{
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", x_" << i;
}
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", y_" << i;
}
}
if (output_3D_landmarks)
{
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", X_" << i;
}
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", Y_" << i;
}
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", Z_" << i;
}
}
// Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters
if (output_model_params)
{
*output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty";
for (int i = 0; i < num_model_modes; ++i)
{
*output_file << ", p_" << i;
}
}
if (output_AUs)
{
std::sort(au_names_reg.begin(), au_names_reg.end());
for (string reg_name : au_names_reg)
{
*output_file << ", " << reg_name << "_r";
}
std::sort(au_names_class.begin(), au_names_class.end());
for (string class_name : au_names_class)
{
*output_file << ", " << class_name << "_c";
}
}
*output_file << endl;
}
// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,

24
lib/local/Recorder/include/RecorderOpenFace.h
View file

@ -39,7 +39,10 @@
// System includes
#include <vector>
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
namespace Recorder
{
@ -55,9 +58,9 @@ namespace Recorder
// The constructor for the recorder, need to specify if we are recording a sequence or not
RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose,
bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg, const std::string& output_codec, double fps_vid_in);
// Simplified constructor that records all
// Simplified constructor that records all, TODO implement
RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
@ -92,6 +95,8 @@ namespace Recorder
// HOG feature related observations
void SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
void SetObservationVisualization(const cv::Mat_<double> &vis_track);
void WriteObservation();
private:
@ -124,8 +129,8 @@ namespace Recorder
// Facial landmark related observations
cv::Mat_<double> landmarks_2D;
cv::Mat_<double> landmarks_3D;
cv::Vec6d params_global;
cv::Mat_<double> params_local;
cv::Vec6d pdm_params_global;
cv::Mat_<double> pdm_params_local;
double landmark_detection_confidence;
bool landmark_detection_success;
@ -137,13 +142,20 @@ namespace Recorder
std::vector<std::pair<std::string, double> > au_occurences;
// Gaze related observations
cv::Point3f gazeDirection0;
cv::Point3f gazeDirection1;
cv::Point3f gaze_direction0;
cv::Point3f gaze_direction1;
cv::Vec2d gaze_angle;
cv::Mat_<double> eye_landmarks;
int observation_count;
// For video writing
cv::VideoWriter video_writer;
std::string video_filename;
std::string output_codec;
double fps_vid_out;
cv::Mat_<double> vis_to_out;
};
}
#endif

5
lib/local/Recorder/src/RecorderCSV.cpp
View file

@ -36,6 +36,9 @@
// For sorting
#include <algorithm>
// For standard out
#include <iostream>
using namespace Recorder;
// Default constructor initializes the variables
@ -45,7 +48,7 @@ RecorderCSV::RecorderCSV():output_file(){};
// Opening the file and preparing the header for it
bool RecorderCSV::Open(std::string output_file_name, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string> au_names_class, const std::vector<std::string> au_names_reg)
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg)
{
output_file.open(output_file_name, std::ios_base::out);

66
lib/local/Recorder/src/RecorderOpenFace.cpp
View file

@ -50,6 +50,9 @@ using namespace boost::filesystem;
using namespace Recorder;
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
void CreateDirectory(std::string output_path)
{
@ -70,10 +73,10 @@ void CreateDirectory(std::string output_path)
RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose,
bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg):
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg, const std::string& output_codec, double fps_vid_in):
is_sequence(sequence), output_2D_landmarks(output_2D_landmarks), output_3D_landmarks(output_3D_landmarks), output_aligned_faces(output_aligned_faces),
output_AUs(output_AUs), output_gaze(output_gaze), output_hog(output_hog), output_model_params(output_model_params),
output_pose(output_pose), output_tracked_video(output_tracked_video)
output_pose(output_pose), output_tracked_video(output_tracked_video), video_writer(), fps_vid_out(fps_vid_in), output_codec(output_codec)
{
// From the filename, strip out the name without directory and extension
@ -96,6 +99,13 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s
}
// TODO construct a video recorder
// saving the videos
if (output_tracked_video)
{
this->video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string();
}
// Prepare image recording
@ -103,14 +113,52 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s
}
void RecorderOpenFace::SetObservationVisualization(const cv::Mat_<double> &vis_track)
{
if (output_tracked_video)
{
// Initialize the video writer if it has not been opened yet
if(!video_writer.isOpened())
{
std::string video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string();
try
{
video_writer.open(video_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), fps_vid_out, vis_track.size(), true);
}
catch (cv::Exception e)
{
WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
}
}
vis_to_out = vis_track;
}
}
void RecorderOpenFace::WriteObservation()
{
observation_count++;
// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success;
this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success,
landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks, au_intensities, au_occurences);
// TODO HOG
if(output_tracked_video)
{
if (vis_to_out.empty)
{
WARN_STREAM("Output tracked video frame is not set");
}
video_writer.write(vis_to_out);
// Clear the output
vis_to_out = cv::Mat();
}
}
@ -125,12 +173,12 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp)
}
void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D,
const cv::Vec6d& params_global, const cv::Mat_<double>& params_local, double confidence, bool success)
const cv::Vec6d& pdm_params_global, const cv::Mat_<double>& pdm_params_local, double confidence, bool success)
{
this->landmarks_2D = landmarks_2D;
this->landmarks_3D = landmarks_3D;
this->params_global = params_global;
this->params_local = params_local;
this->pdm_params_global = pdm_params_global;
this->pdm_params_local = pdm_params_local;
this->landmark_detection_confidence = confidence;
this->landmark_detection_success = success;
@ -148,11 +196,11 @@ void RecorderOpenFace::SetObservationActionUnits(const std::vector<std::pair<std
this->au_occurences = au_occurences;
}
void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1,
void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1,
const cv::Vec2d& gaze_angle, const cv::Mat_<double>& eye_landmarks)
{
this->gazeDirection0 = gazeDirection0;
this->gazeDirection1 = gazeDirection1;
this->gaze_direction0 = gaze_direction0;
this->gaze_direction1 = gaze_direction1;
this->gaze_angle = gaze_angle;
this->eye_landmarks = eye_landmarks;
}