r
/
sustaining_gazes
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Interface changes to provide multi output support, also enforcing parameters in executables.

This commit is contained in:
Tadas Baltrusaitis 2018-02-23 08:25:19 +00:00
parent aba6ea646b
commit a4de1d42a5
11 changed files with 179 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
exe/FaceLandmarkImg/FaceLandmarkImg.cpp
View File

@ -85,6 +85,14 @@ int main (int argc, char **argv)
//Convert arguments to more convenient vector form
vector<string> arguments = get_arguments(argc, argv);
// no arguments: output usage
if (arguments.size() == 1)
{
cout << "For command line arguments see:" << endl;
cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
// Prepare for image reading
Utilities::ImageCapture image_reader;
@ -205,6 +213,7 @@ int main (int argc, char **argv)
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy));
open_face_rec.SetObservationFaceAlign(sim_warped_img);
open_face_rec.SetObservationFaceID(face);
open_face_rec.WriteObservation();
}

123
exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp
View File

@ -39,6 +39,10 @@
#include "VisualizationUtils.h"
#include "Visualizer.h"
#include "SequenceCapture.h"
#include <RecorderOpenFace.h>
#include <RecorderOpenFaceParameters.h>
#include <GazeEstimation.h>
#include <FaceAnalyser.h>
#include <fstream>
#include <sstream>
@ -110,7 +114,14 @@ int main (int argc, char **argv)
vector<string> arguments = get_arguments(argc, argv);
// no arguments: output usage
if (arguments.size() == 1)
{
cout << "For command line arguments see:" << endl;
cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
LandmarkDetector::FaceModelParameters det_params(arguments);
// This is so that the model would not try re-initialising itself
det_params.reinit_video_every = -1;
@ -142,11 +153,16 @@ int main (int argc, char **argv)
det_parameters.push_back(det_params);
}
// Load facial feature extractor and AU analyser (make sure it is static, as we don't reidentify faces)
FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
face_analysis_params.OptimizeForImages();
FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
// Open a sequence
Utilities::SequenceCapture sequence_reader;
// A utility for visualizing the results (show just the tracks)
Utilities::Visualizer visualizer(true, false, false);
Utilities::Visualizer visualizer(arguments);
// Tracking FPS for visualization
Utilities::FpsTracker fps_tracker;
@ -159,46 +175,41 @@ int main (int argc, char **argv)
// The sequence reader chooses what to open based on command line arguments provided
if (!sequence_reader.Open(arguments))
{
// If failed to open because no input files specified, attempt to open a webcam
if (sequence_reader.no_input_specified && sequence_number == 0)
{
// If that fails, revert to webcam
INFO_STREAM("No input specified, attempting to open a webcam 0");
if (!sequence_reader.OpenWebcam(0))
{
ERROR_STREAM("Failed to open the webcam");
break;
}
}
else
{
break;
}
}
break;
INFO_STREAM("Device or file opened");
cv::Mat captured_image = sequence_reader.GetNextFrame();
int frame_count = 0;
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(),
sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps);
// Do not do AU detection on multi-face case as it is not supported
recording_params.setOutputAUs(false);
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
if (recording_params.outputGaze() && !face_model.eye_model)
cout << "WARNING: no eye model defined, but outputting gaze" << endl;
if (sequence_reader.IsWebcam())
{
INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)");
visualizer.vis_track = true;
}
if (recording_params.outputAUs())
{
INFO_STREAM("WARNING: using a AU detection in multiple face mode, it might not be as accurate and is experimental");
}
INFO_STREAM( "Starting tracking");
while(!captured_image.empty())
{
// Reading the images
cv::Mat_<uchar> grayscale_image;
cv::Mat disp_image = captured_image.clone();
if(captured_image.channels() == 3)
{
cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY);
}
else
{
grayscale_image = captured_image.clone();
}
cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
vector<cv::Rect_<double> > face_detections;
@ -261,7 +272,7 @@ int main (int argc, char **argv)
// This ensures that a wider window is used for the initial landmark localisation
face_models[model].detection_success = false;
detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], face_models[model], det_parameters[model]);
// This activates the model
active_models[model] = true;
@ -283,14 +294,60 @@ int main (int argc, char **argv)
visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// Go through every model and visualise the results
// Go through every model and detect eye gaze, record results and visualise the results
for(size_t model = 0; model < face_models.size(); ++model)
{
// Visualising the results
if(active_models[model])
{
// Estimate head pose and eye gaze
cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
cv::Point3f gaze_direction0(0, 0, 0); cv::Point3f gaze_direction1(0, 0, 0); cv::Vec2d gaze_angle(0, 0);
// Detect eye gazes
if (face_models[model].detection_success && face_model.eye_model)
{
GazeAnalysis::EstimateGaze(face_models[model], gaze_direction0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(face_models[model], gaze_direction1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
gaze_angle = GazeAnalysis::GetGazeAngle(gaze_direction0, gaze_direction1);
}
// Face analysis step
cv::Mat sim_warped_img;
cv::Mat_<double> hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0;
// Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization
if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog)
{
face_analyser.PredictStaticAUsAndComputeFeatures(captured_image, face_models[model].detected_landmarks);
face_analyser.GetLatestAlignedFace(sim_warped_img);
face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols);
}
// Visualize the features
visualizer.SetObservationFaceAlign(sim_warped_img);
visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty);
visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty);
visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty);
// Output features
open_face_rec.SetObservationHOG(face_models[model].detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG
open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
open_face_rec.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
face_models[model].params_global, face_models[model].params_local, face_models[model].detection_certainty, face_models[model].detection_success);
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
open_face_rec.SetObservationFaceAlign(sim_warped_img);
open_face_rec.SetObservationFaceID(model);
open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber());
open_face_rec.WriteObservation();
}
}
visualizer.SetFps(fps_tracker.GetFPS());

10
exe/FeatureExtraction/FeatureExtraction.cpp
View File

@ -103,6 +103,14 @@ int main (int argc, char **argv)
vector<string> arguments = get_arguments(argc, argv);
// no arguments: output usage
if (arguments.size() == 1)
{
cout << "For command line arguments see:" << endl;
cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
// Load the modules that are being used for tracking and face analysis
// Load face landmark detector
LandmarkDetector::FaceModelParameters det_parameters(arguments);
@ -217,6 +225,8 @@ int main (int argc, char **argv)
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
open_face_rec.SetObservationFaceID(0);
open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber());
open_face_rec.SetObservationFaceAlign(sim_warped_img);
open_face_rec.WriteObservation();

4
lib/local/Utilities/include/RecorderCSV.h
View File

@ -60,10 +60,12 @@ namespace Utilities
bool Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
bool isOpen() const { return output_file.is_open(); }
// Closing the file and cleaning up
void Close();
void WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence,
void WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D, const cv::Mat_<double>& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate,
const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d,
const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);

12
lib/local/Utilities/include/RecorderOpenFace.h
View File

@ -70,6 +70,12 @@ namespace Utilities
// Required observations for video/image-sequence
void SetObservationTimestamp(double timestamp);
// Required observations for video/image-sequence
void SetObservationFrameNumber(double frame_number);
// If in multiple face mode, identifying which face was tracked
void SetObservationFaceID(int face_id);
// All observations relevant to facial landmarks
void SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D,
const cv::Vec6d& params_global, const cv::Mat_<double>& params_local, double confidence, bool success);
@ -123,7 +129,10 @@ namespace Utilities
RecorderHOG hog_recorder;
// The actual temporary storage for the observations
double timestamp;
int face_id;
int frame_number;
// Facial landmark related observations
cv::Mat_<double> landmarks_2D;
@ -147,7 +156,8 @@ namespace Utilities
std::vector<cv::Point2d> eye_landmarks2D;
std::vector<cv::Point3d> eye_landmarks3D;
int observation_count;
int m_frame_number; // TODO this should be set
int m_face_id; // TODO this should be set
// For video writing
cv::VideoWriter video_writer;

2
lib/local/Utilities/include/RecorderOpenFaceParameters.h
View File

@ -77,6 +77,8 @@ namespace Utilities
float getCx() const { return cx; }
float getCy() const { return cy; }
void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; }
private:
// If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face

2
lib/local/Utilities/include/SequenceCapture.h
View File

@ -85,6 +85,8 @@ namespace Utilities
// Parameters describing the sequence and it's progress
double GetProgress();
int GetFrameNumber() { return frame_num; }
bool IsOpened();
void Close();

9
lib/local/Utilities/src/RecorderCSV.cpp
View File

@ -78,7 +78,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp
// Different headers if we are writing out the results on a sequence or an individual image
if(this->is_sequence)
{
output_file << "frame, timestamp, confidence, success";
output_file << "frame, face_id, timestamp, confidence, success";
}
else
{
@ -176,7 +176,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp
}
void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence,
void RecorderCSV::WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D, const cv::Mat_<double>& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate,
const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d,
const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences)
@ -193,8 +193,9 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm
output_file << std::noshowpoint;
if(is_sequence)
{
output_file << std::setprecision(3);
output_file << observation_count << ", " << time_stamp;
output_file << frame_num << ", " << face_id << ", " << time_stamp;
output_file << std::setprecision(2);
output_file << ", " << landmark_confidence;
output_file << std::setprecision(0);
@ -203,7 +204,7 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm
else
{
output_file << std::setprecision(3);
output_file << observation_count << ", " << landmark_confidence;
output_file << face_id << ", " << landmark_confidence;
}
// Output the estimated gaze
if (output_gaze)

27
lib/local/Utilities/src/RecorderOpenFace.cpp
View File

@ -153,7 +153,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename)
CreateDirectory(aligned_output_directory);
}
observation_count = 0;
this->frame_number = 0;
}
@ -275,11 +275,9 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track)
void RecorderOpenFace::WriteObservation()
{
observation_count++;
// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
if(observation_count == 1)
// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
if(!csv_recorder.isOpen())
{
// As we are writing out the header, work out some things like number of landmarks, names of AUs etc.
int num_face_landmarks = landmarks_2D.rows / 2;
@ -315,7 +313,7 @@ void RecorderOpenFace::WriteObservation()
params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg);
}
this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success,
this->csv_recorder.WriteLine(face_id, frame_number, timestamp, landmark_detection_success,
landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences);
@ -331,9 +329,9 @@ void RecorderOpenFace::WriteObservation()
// Filename is based on frame number
if(params.isSequence())
std::sprintf(name, "frame_det_%06d.bmp", observation_count);
std::sprintf(name, "frame_det_%02d_%06d.bmp", face_id, frame_number);
else
std::sprintf(name, "face_det_%06d.bmp", observation_count);
std::sprintf(name, "face_det_%06d.bmp", face_id);
// Construct the output filename
boost::filesystem::path slash("/");
@ -387,6 +385,19 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp)
this->timestamp = timestamp;
}
// Required observations for video/image-sequence
void RecorderOpenFace::SetObservationFrameNumber(double frame_number)
{
this->frame_number = frame_number;
}
// If in multiple face mode, identifying which face was tracked
void RecorderOpenFace::SetObservationFaceID(int face_id)
{
this->face_id = face_id;
}
void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, const cv::Mat_<double>& landmarks_3D,
const cv::Vec6d& pdm_params_global, const cv::Mat_<double>& pdm_params_local, double confidence, bool success)
{

54
lib/local/Utilities/src/RecorderOpenFaceParameters.cpp
View File

@ -60,61 +60,61 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector<std::string>
bool output_set = false;
output_2D_landmarks = false;
output_3D_landmarks = false;
output_model_params = false;
output_pose = false;
output_AUs = false;
output_gaze = false;
output_hog = false;
output_tracked = false;
output_aligned_faces = false;
this->output_2D_landmarks = false;
this->output_3D_landmarks = false;
this->output_model_params = false;
this->output_pose = false;
this->output_AUs = false;
this->output_gaze = false;
this->output_hog = false;
this->output_tracked = false;
this->output_aligned_faces = false;
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-simalign") == 0)
{
output_aligned_faces = true;
this->output_aligned_faces = true;
output_set = true;
}
else if (arguments[i].compare("-hogalign") == 0)
{
output_hog = true;
this->output_hog = true;
output_set = true;
}
else if (arguments[i].compare("-2Dfp") == 0)
{
output_2D_landmarks = true;
this->output_2D_landmarks = true;
output_set = true;
}
else if (arguments[i].compare("-3Dfp") == 0)
{
output_3D_landmarks = true;
this->output_3D_landmarks = true;
output_set = true;
}
else if (arguments[i].compare("-pdmparams") == 0)
{
output_model_params = true;
this->output_model_params = true;
output_set = true;
}
else if (arguments[i].compare("-pose") == 0)
{
output_pose = true;
this->output_pose = true;
output_set = true;
}
else if (arguments[i].compare("-aus") == 0)
{
output_AUs = true;
this->output_AUs = true;
output_set = true;
}
else if (arguments[i].compare("-gaze") == 0)
{
output_gaze = true;
this->output_gaze = true;
output_set = true;
}
else if (arguments[i].compare("-tracked") == 0)
{
output_tracked = true;
this->output_tracked = true;
output_set = true;
}
}
@ -123,15 +123,15 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector<std::string>
if (!output_set)
{
output_2D_landmarks = true;
output_3D_landmarks = true;
output_model_params = true;
output_pose = true;
output_AUs = true;
output_gaze = true;
output_hog = true;
output_tracked = true;
output_aligned_faces = true;
this->output_2D_landmarks = true;
this->output_3D_landmarks = true;
this->output_model_params = true;
this->output_pose = true;
this->output_AUs = true;
this->output_gaze = true;
this->output_hog = true;
this->output_tracked = true;
this->output_aligned_faces = true;
}
}

2
matlab_runners/Demos/run_demo_video_multi.m
View File

@ -19,7 +19,7 @@ model = 'model/main_clnf_general.txt'; % Trained on in the wild and multi-pie da
%model = 'model/main_clm_wild.txt'; % Trained on in-the-wild
% Create a command that will run the tracker on set of videos and display the output
command = sprintf('%s -mloc "%s" ', executable, model);
command = sprintf('%s -mloc "%s" -verbose ', executable, model);
% add all videos to single argument list (so as not to load the model anew
% for every video)