Continuing work on the recorder.
This commit is contained in:
Tadas Baltrusaitis
parent
4f4bf263d6
commit
b60669fa62
|
|
@ -203,10 +203,6 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
|
|||
}
|
||||
}
|
||||
|
||||
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg);
|
||||
|
||||
// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
|
||||
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
|
|
@ -224,7 +220,8 @@ int main (int argc, char **argv)
|
|||
|
||||
// Get the input output file parameters
|
||||
|
||||
string output_codec; //not used but should
|
||||
string output_codec;
|
||||
// TODO rename
|
||||
LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, output_codec, arguments);
|
||||
|
||||
bool video_input = true;
|
||||
|
|
@ -312,6 +309,7 @@ int main (int argc, char **argv)
|
|||
|
||||
double fps_vid_in = -1.0;
|
||||
|
||||
// TODO this should be moved to a SequenceCapture class
|
||||
if(video_input)
|
||||
{
|
||||
// We might specify multiple video files as arguments
|
||||
|
|
@ -391,27 +389,8 @@ int main (int argc, char **argv)
|
|||
// TODO this should always be video input
|
||||
int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); // TODO empty file check replaced
|
||||
Recorder::RecorderOpenFace openFaceRec(output_files[f_n], input_files[f_n], true, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, !output_hog_align_files.empty(),
|
||||
!tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
|
||||
|
||||
// Creating output files
|
||||
std::ofstream output_file;
|
||||
|
||||
if (!output_files.empty())
|
||||
{
|
||||
output_file.open(output_files[f_n], ios_base::out);
|
||||
int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size();
|
||||
!tracked_videos_output.empty(), !output_similarity_align.empty(), face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames(), output_codec, fps_vid_in);
|
||||
|
||||
prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
|
||||
}
|
||||
|
||||
|
||||
// Saving the HOG features
|
||||
std::ofstream hog_output_file;
|
||||
if(!output_hog_align_files.empty())
|
||||
{
|
||||
hog_output_file.open(output_hog_align_files[f_n], ios_base::out | ios_base::binary);
|
||||
}
|
||||
|
||||
// saving the videos
|
||||
cv::VideoWriter writerFace;
|
||||
if(!tracked_videos_output.empty())
|
||||
|
|
@ -424,19 +403,10 @@ int main (int argc, char **argv)
|
|||
{
|
||||
WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
int frame_count = 0;
|
||||
|
||||
// This is useful for a second pass run (if want AU predictions)
|
||||
// TODO remove these
|
||||
vector<cv::Vec6d> params_global_video;
|
||||
vector<bool> successes_video;
|
||||
vector<cv::Mat_<double>> params_local_video;
|
||||
vector<cv::Mat_<double>> detected_landmarks_video;
|
||||
|
||||
|
||||
// Use for timestamping if using a webcam
|
||||
int64 t_initial = cv::getTickCount();
|
||||
|
||||
|
|
@ -649,89 +619,6 @@ int main (int argc, char **argv)
|
|||
return 0;
|
||||
}
|
||||
|
||||
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg)
|
||||
{
|
||||
|
||||
*output_file << "frame, timestamp, confidence, success";
|
||||
|
||||
if (output_gaze)
|
||||
{
|
||||
*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
|
||||
|
||||
for (int i = 0; i < num_eye_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", eye_lmk_x_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_eye_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", eye_lmk_y_" << i;
|
||||
}
|
||||
}
|
||||
|
||||
if (output_pose)
|
||||
{
|
||||
*output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz";
|
||||
}
|
||||
|
||||
if (output_2D_landmarks)
|
||||
{
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", x_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", y_" << i;
|
||||
}
|
||||
}
|
||||
|
||||
if (output_3D_landmarks)
|
||||
{
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", X_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", Y_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", Z_" << i;
|
||||
}
|
||||
}
|
||||
|
||||
// Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters
|
||||
if (output_model_params)
|
||||
{
|
||||
*output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty";
|
||||
for (int i = 0; i < num_model_modes; ++i)
|
||||
{
|
||||
*output_file << ", p_" << i;
|
||||
}
|
||||
}
|
||||
|
||||
if (output_AUs)
|
||||
{
|
||||
std::sort(au_names_reg.begin(), au_names_reg.end());
|
||||
for (string reg_name : au_names_reg)
|
||||
{
|
||||
*output_file << ", " << reg_name << "_r";
|
||||
}
|
||||
|
||||
std::sort(au_names_class.begin(), au_names_class.end());
|
||||
for (string class_name : au_names_class)
|
||||
{
|
||||
*output_file << ", " << class_name << "_c";
|
||||
}
|
||||
}
|
||||
|
||||
*output_file << endl;
|
||||
|
||||
}
|
||||
|
||||
// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
|
||||
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
|
|
|
|||
|
|
@ -39,7 +39,10 @@
|
|||
|
||||
// System includes
|
||||
#include <vector>
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
|
||||
namespace Recorder
|
||||
{
|
||||
|
|
@ -55,9 +58,9 @@ namespace Recorder
|
|||
// The constructor for the recorder, need to specify if we are recording a sequence or not
|
||||
RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose,
|
||||
bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
|
||||
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
|
||||
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg, const std::string& output_codec, double fps_vid_in);
|
||||
|
||||
// Simplified constructor that records all
|
||||
// Simplified constructor that records all, TODO implement
|
||||
RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
|
||||
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
|
||||
|
||||
|
|
@ -92,6 +95,8 @@ namespace Recorder
|
|||
// HOG feature related observations
|
||||
void SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
|
||||
|
||||
void SetObservationVisualization(const cv::Mat_<double> &vis_track);
|
||||
|
||||
void WriteObservation();
|
||||
|
||||
private:
|
||||
|
|
@ -124,8 +129,8 @@ namespace Recorder
|
|||
// Facial landmark related observations
|
||||
cv::Mat_<double> landmarks_2D;
|
||||
cv::Mat_<double> landmarks_3D;
|
||||
cv::Vec6d params_global;
|
||||
cv::Mat_<double> params_local;
|
||||
cv::Vec6d pdm_params_global;
|
||||
cv::Mat_<double> pdm_params_local;
|
||||
double landmark_detection_confidence;
|
||||
bool landmark_detection_success;
|
||||
|
||||
|
|
@ -137,13 +142,20 @@ namespace Recorder
|
|||
std::vector<std::pair<std::string, double> > au_occurences;
|
||||
|
||||
// Gaze related observations
|
||||
cv::Point3f gazeDirection0;
|
||||
cv::Point3f gazeDirection1;
|
||||
cv::Point3f gaze_direction0;
|
||||
cv::Point3f gaze_direction1;
|
||||
cv::Vec2d gaze_angle;
|
||||
cv::Mat_<double> eye_landmarks;
|
||||
|
||||
int observation_count;
|
||||
|
||||
// For video writing
|
||||
cv::VideoWriter video_writer;
|
||||
std::string video_filename;
|
||||
std::string output_codec;
|
||||
double fps_vid_out;
|
||||
cv::Mat_<double> vis_to_out;
|
||||
|
||||
};
|
||||
}
|
||||
#endif
|
||||
|
|
@ -36,6 +36,9 @@
|
|||
// For sorting
|
||||
#include <algorithm>
|
||||
|
||||
// For standard out
|
||||
#include <iostream>
|
||||
|
||||
using namespace Recorder;
|
||||
|
||||
// Default constructor initializes the variables
|
||||
|
|
@ -45,7 +48,7 @@ RecorderCSV::RecorderCSV():output_file(){};
|
|||
|
||||
// Opening the file and preparing the header for it
|
||||
bool RecorderCSV::Open(std::string output_file_name, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string> au_names_class, const std::vector<std::string> au_names_reg)
|
||||
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg)
|
||||
{
|
||||
output_file.open(output_file_name, std::ios_base::out);
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,9 @@ using namespace boost::filesystem;
|
|||
|
||||
using namespace Recorder;
|
||||
|
||||
#define WARN_STREAM( stream ) \
|
||||
std::cout << "Warning: " << stream << std::endl
|
||||
|
||||
void CreateDirectory(std::string output_path)
|
||||
{
|
||||
|
||||
|
|
@ -70,10 +73,10 @@ void CreateDirectory(std::string output_path)
|
|||
|
||||
RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::string in_filename, bool sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose,
|
||||
bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked_video, bool output_aligned_faces, int num_face_landmarks, int num_model_modes, int num_eye_landmarks,
|
||||
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg):
|
||||
const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg, const std::string& output_codec, double fps_vid_in):
|
||||
is_sequence(sequence), output_2D_landmarks(output_2D_landmarks), output_3D_landmarks(output_3D_landmarks), output_aligned_faces(output_aligned_faces),
|
||||
output_AUs(output_AUs), output_gaze(output_gaze), output_hog(output_hog), output_model_params(output_model_params),
|
||||
output_pose(output_pose), output_tracked_video(output_tracked_video)
|
||||
output_pose(output_pose), output_tracked_video(output_tracked_video), video_writer(), fps_vid_out(fps_vid_in), output_codec(output_codec)
|
||||
{
|
||||
|
||||
// From the filename, strip out the name without directory and extension
|
||||
|
|
@ -96,6 +99,13 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s
|
|||
}
|
||||
|
||||
// TODO construct a video recorder
|
||||
// saving the videos
|
||||
|
||||
if (output_tracked_video)
|
||||
{
|
||||
this->video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string();
|
||||
}
|
||||
|
||||
|
||||
// Prepare image recording
|
||||
|
||||
|
|
@ -103,14 +113,52 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s
|
|||
|
||||
}
|
||||
|
||||
void RecorderOpenFace::SetObservationVisualization(const cv::Mat_<double> &vis_track)
|
||||
{
|
||||
if (output_tracked_video)
|
||||
{
|
||||
// Initialize the video writer if it has not been opened yet
|
||||
if(!video_writer.isOpened())
|
||||
{
|
||||
std::string video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string();
|
||||
try
|
||||
{
|
||||
video_writer.open(video_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), fps_vid_out, vis_track.size(), true);
|
||||
}
|
||||
catch (cv::Exception e)
|
||||
{
|
||||
WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
|
||||
}
|
||||
}
|
||||
|
||||
vis_to_out = vis_track;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void RecorderOpenFace::WriteObservation()
|
||||
{
|
||||
observation_count++;
|
||||
|
||||
// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
|
||||
this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success;
|
||||
this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success,
|
||||
landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
|
||||
gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks, au_intensities, au_occurences);
|
||||
|
||||
// TODO HOG
|
||||
|
||||
if(output_tracked_video)
|
||||
{
|
||||
if (vis_to_out.empty)
|
||||
{
|
||||
WARN_STREAM("Output tracked video frame is not set");
|
||||
}
|
||||
video_writer.write(vis_to_out);
|
||||
// Clear the output
|
||||
vis_to_out = cv::Mat();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -125,12 +173,12 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp)
|
|||
}
|
||||
|
||||
void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D,
|
||||
const cv::Vec6d& params_global, const cv::Mat_<double>& params_local, double confidence, bool success)
|
||||
const cv::Vec6d& pdm_params_global, const cv::Mat_<double>& pdm_params_local, double confidence, bool success)
|
||||
{
|
||||
this->landmarks_2D = landmarks_2D;
|
||||
this->landmarks_3D = landmarks_3D;
|
||||
this->params_global = params_global;
|
||||
this->params_local = params_local;
|
||||
this->pdm_params_global = pdm_params_global;
|
||||
this->pdm_params_local = pdm_params_local;
|
||||
this->landmark_detection_confidence = confidence;
|
||||
this->landmark_detection_success = success;
|
||||
|
||||
|
|
@ -148,11 +196,11 @@ void RecorderOpenFace::SetObservationActionUnits(const std::vector<std::pair<std
|
|||
this->au_occurences = au_occurences;
|
||||
}
|
||||
|
||||
void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1,
|
||||
void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1,
|
||||
const cv::Vec2d& gaze_angle, const cv::Mat_<double>& eye_landmarks)
|
||||
{
|
||||
this->gazeDirection0 = gazeDirection0;
|
||||
this->gazeDirection1 = gazeDirection1;
|
||||
this->gaze_direction0 = gaze_direction0;
|
||||
this->gaze_direction1 = gaze_direction1;
|
||||
this->gaze_angle = gaze_angle;
|
||||
this->eye_landmarks = eye_landmarks;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue