diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 9448cf3..594ac7d 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -85,6 +85,14 @@ int main (int argc, char **argv) //Convert arguments to more convenient vector form vector arguments = get_arguments(argc, argv); + // no arguments: output usage + if (arguments.size() == 1) + { + cout << "For command line arguments see:" << endl; + cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments"; + return 0; + } + // Prepare for image reading Utilities::ImageCapture image_reader; @@ -205,6 +213,7 @@ int main (int argc, char **argv) open_face_rec.SetObservationPose(pose_estimate); open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy)); open_face_rec.SetObservationFaceAlign(sim_warped_img); + open_face_rec.SetObservationFaceID(face); open_face_rec.WriteObservation(); } diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index cbb1e09..2c7d7e5 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -39,6 +39,10 @@ #include "VisualizationUtils.h" #include "Visualizer.h" #include "SequenceCapture.h" +#include +#include +#include +#include #include #include @@ -110,7 +114,14 @@ int main (int argc, char **argv) vector arguments = get_arguments(argc, argv); - + // no arguments: output usage + if (arguments.size() == 1) + { + cout << "For command line arguments see:" << endl; + cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments"; + return 0; + } + LandmarkDetector::FaceModelParameters det_params(arguments); // This is so that the model would not try re-initialising itself det_params.reinit_video_every = -1; @@ -142,11 +153,16 @@ int main (int argc, char **argv) det_parameters.push_back(det_params); } + // Load facial feature extractor and AU analyser (make sure it is static, as we don't reidentify faces) + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); + face_analysis_params.OptimizeForImages(); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); + // Open a sequence Utilities::SequenceCapture sequence_reader; // A utility for visualizing the results (show just the tracks) - Utilities::Visualizer visualizer(true, false, false); + Utilities::Visualizer visualizer(arguments); // Tracking FPS for visualization Utilities::FpsTracker fps_tracker; @@ -159,46 +175,41 @@ int main (int argc, char **argv) // The sequence reader chooses what to open based on command line arguments provided if (!sequence_reader.Open(arguments)) - { - // If failed to open because no input files specified, attempt to open a webcam - if (sequence_reader.no_input_specified && sequence_number == 0) - { - // If that fails, revert to webcam - INFO_STREAM("No input specified, attempting to open a webcam 0"); - if (!sequence_reader.OpenWebcam(0)) - { - ERROR_STREAM("Failed to open the webcam"); - break; - } - } - else - { - break; - } - } + break; + INFO_STREAM("Device or file opened"); cv::Mat captured_image = sequence_reader.GetNextFrame(); int frame_count = 0; + Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), + sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); + // Do not do AU detection on multi-face case as it is not supported + recording_params.setOutputAUs(false); + Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); + + if (recording_params.outputGaze() && !face_model.eye_model) + cout << "WARNING: no eye model defined, but outputting gaze" << endl; + + if (sequence_reader.IsWebcam()) + { + INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)"); + visualizer.vis_track = true; + } + + if (recording_params.outputAUs()) + { + INFO_STREAM("WARNING: using a AU detection in multiple face mode, it might not be as accurate and is experimental"); + } + + INFO_STREAM( "Starting tracking"); while(!captured_image.empty()) { // Reading the images - cv::Mat_ grayscale_image; - - cv::Mat disp_image = captured_image.clone(); - - if(captured_image.channels() == 3) - { - cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); - } - else - { - grayscale_image = captured_image.clone(); - } + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); vector > face_detections; @@ -261,7 +272,7 @@ int main (int argc, char **argv) // This ensures that a wider window is used for the initial landmark localisation face_models[model].detection_success = false; detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], face_models[model], det_parameters[model]); - + // This activates the model active_models[model] = true; @@ -283,14 +294,60 @@ int main (int argc, char **argv) visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - // Go through every model and visualise the results + // Go through every model and detect eye gaze, record results and visualise the results for(size_t model = 0; model < face_models.size(); ++model) { // Visualising the results if(active_models[model]) { + + // Estimate head pose and eye gaze + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + + cv::Point3f gaze_direction0(0, 0, 0); cv::Point3f gaze_direction1(0, 0, 0); cv::Vec2d gaze_angle(0, 0); + + // Detect eye gazes + if (face_models[model].detection_success && face_model.eye_model) + { + GazeAnalysis::EstimateGaze(face_models[model], gaze_direction0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); + GazeAnalysis::EstimateGaze(face_models[model], gaze_direction1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); + gaze_angle = GazeAnalysis::GetGazeAngle(gaze_direction0, gaze_direction1); + } + + // Face analysis step + cv::Mat sim_warped_img; + cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; + + // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) + { + face_analyser.PredictStaticAUsAndComputeFeatures(captured_image, face_models[model].detected_landmarks); + face_analyser.GetLatestAlignedFace(sim_warped_img); + face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); + } + + // Visualize the features + visualizer.SetObservationFaceAlign(sim_warped_img); + visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty); visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty); + visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty); + + // Output features + open_face_rec.SetObservationHOG(face_models[model].detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG + open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); + open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + open_face_rec.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), + face_models[model].params_global, face_models[model].params_local, face_models[model].detection_certainty, face_models[model].detection_success); + open_face_rec.SetObservationPose(pose_estimate); + open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); + open_face_rec.SetObservationFaceAlign(sim_warped_img); + open_face_rec.SetObservationFaceID(model); + open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); + open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber()); + open_face_rec.WriteObservation(); + + } } visualizer.SetFps(fps_tracker.GetFPS()); diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index a3c9e04..b074d9f 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -103,6 +103,14 @@ int main (int argc, char **argv) vector arguments = get_arguments(argc, argv); + // no arguments: output usage + if (arguments.size() == 1) + { + cout << "For command line arguments see:" << endl; + cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments"; + return 0; + } + // Load the modules that are being used for tracking and face analysis // Load face landmark detector LandmarkDetector::FaceModelParameters det_parameters(arguments); @@ -217,6 +225,8 @@ int main (int argc, char **argv) open_face_rec.SetObservationPose(pose_estimate); open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); + open_face_rec.SetObservationFaceID(0); + open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber()); open_face_rec.SetObservationFaceAlign(sim_warped_img); open_face_rec.WriteObservation(); diff --git a/lib/local/Utilities/include/RecorderCSV.h b/lib/local/Utilities/include/RecorderCSV.h index 7a625d5..892fb9e 100644 --- a/lib/local/Utilities/include/RecorderCSV.h +++ b/lib/local/Utilities/include/RecorderCSV.h @@ -60,10 +60,12 @@ namespace Utilities bool Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector& au_names_class, const std::vector& au_names_reg); + bool isOpen() const { return output_file.is_open(); } + // Closing the file and cleaning up void Close(); - void WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence, + void WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence, const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Mat_& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate, const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, const std::vector >& au_intensities, const std::vector >& au_occurences); diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h index a8b4b51..bb13d7b 100644 --- a/lib/local/Utilities/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -70,6 +70,12 @@ namespace Utilities // Required observations for video/image-sequence void SetObservationTimestamp(double timestamp); + // Required observations for video/image-sequence + void SetObservationFrameNumber(double frame_number); + + // If in multiple face mode, identifying which face was tracked + void SetObservationFaceID(int face_id); + // All observations relevant to facial landmarks void SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Vec6d& params_global, const cv::Mat_& params_local, double confidence, bool success); @@ -123,7 +129,10 @@ namespace Utilities RecorderHOG hog_recorder; // The actual temporary storage for the observations + double timestamp; + int face_id; + int frame_number; // Facial landmark related observations cv::Mat_ landmarks_2D; @@ -147,7 +156,8 @@ namespace Utilities std::vector eye_landmarks2D; std::vector eye_landmarks3D; - int observation_count; + int m_frame_number; // TODO this should be set + int m_face_id; // TODO this should be set // For video writing cv::VideoWriter video_writer; diff --git a/lib/local/Utilities/include/RecorderOpenFaceParameters.h b/lib/local/Utilities/include/RecorderOpenFaceParameters.h index 2f791e7..8d6b176 100644 --- a/lib/local/Utilities/include/RecorderOpenFaceParameters.h +++ b/lib/local/Utilities/include/RecorderOpenFaceParameters.h @@ -77,6 +77,8 @@ namespace Utilities float getCx() const { return cx; } float getCy() const { return cy; } + void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; } + private: // If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index 422faff..daafbac 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -85,6 +85,8 @@ namespace Utilities // Parameters describing the sequence and it's progress double GetProgress(); + int GetFrameNumber() { return frame_num; } + bool IsOpened(); void Close(); diff --git a/lib/local/Utilities/src/RecorderCSV.cpp b/lib/local/Utilities/src/RecorderCSV.cpp index 3754261..77f4f31 100644 --- a/lib/local/Utilities/src/RecorderCSV.cpp +++ b/lib/local/Utilities/src/RecorderCSV.cpp @@ -78,7 +78,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp // Different headers if we are writing out the results on a sequence or an individual image if(this->is_sequence) { - output_file << "frame, timestamp, confidence, success"; + output_file << "frame, face_id, timestamp, confidence, success"; } else { @@ -176,7 +176,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp } -void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence, +void RecorderCSV::WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence, const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Mat_& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate, const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, const std::vector >& au_intensities, const std::vector >& au_occurences) @@ -193,8 +193,9 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm output_file << std::noshowpoint; if(is_sequence) { + output_file << std::setprecision(3); - output_file << observation_count << ", " << time_stamp; + output_file << frame_num << ", " << face_id << ", " << time_stamp; output_file << std::setprecision(2); output_file << ", " << landmark_confidence; output_file << std::setprecision(0); @@ -203,7 +204,7 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm else { output_file << std::setprecision(3); - output_file << observation_count << ", " << landmark_confidence; + output_file << face_id << ", " << landmark_confidence; } // Output the estimated gaze if (output_gaze) diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 190d7bb..19d88dc 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -153,7 +153,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) CreateDirectory(aligned_output_directory); } - observation_count = 0; + this->frame_number = 0; } @@ -275,11 +275,9 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) void RecorderOpenFace::WriteObservation() { - observation_count++; - // Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers) - - if(observation_count == 1) + // Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers) + if(!csv_recorder.isOpen()) { // As we are writing out the header, work out some things like number of landmarks, names of AUs etc. int num_face_landmarks = landmarks_2D.rows / 2; @@ -315,7 +313,7 @@ void RecorderOpenFace::WriteObservation() params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg); } - this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, + this->csv_recorder.WriteLine(face_id, frame_number, timestamp, landmark_detection_success, landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose, gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences); @@ -331,9 +329,9 @@ void RecorderOpenFace::WriteObservation() // Filename is based on frame number if(params.isSequence()) - std::sprintf(name, "frame_det_%06d.bmp", observation_count); + std::sprintf(name, "frame_det_%02d_%06d.bmp", face_id, frame_number); else - std::sprintf(name, "face_det_%06d.bmp", observation_count); + std::sprintf(name, "face_det_%06d.bmp", face_id); // Construct the output filename boost::filesystem::path slash("/"); @@ -387,6 +385,19 @@ void RecorderOpenFace::SetObservationTimestamp(double timestamp) this->timestamp = timestamp; } +// Required observations for video/image-sequence +void RecorderOpenFace::SetObservationFrameNumber(double frame_number) +{ + this->frame_number = frame_number; +} + +// If in multiple face mode, identifying which face was tracked +void RecorderOpenFace::SetObservationFaceID(int face_id) +{ + this->face_id = face_id; +} + + void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Vec6d& pdm_params_global, const cv::Mat_& pdm_params_local, double confidence, bool success) { diff --git a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp index 2afa45e..259cabd 100644 --- a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp +++ b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp @@ -60,61 +60,61 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector bool output_set = false; - output_2D_landmarks = false; - output_3D_landmarks = false; - output_model_params = false; - output_pose = false; - output_AUs = false; - output_gaze = false; - output_hog = false; - output_tracked = false; - output_aligned_faces = false; + this->output_2D_landmarks = false; + this->output_3D_landmarks = false; + this->output_model_params = false; + this->output_pose = false; + this->output_AUs = false; + this->output_gaze = false; + this->output_hog = false; + this->output_tracked = false; + this->output_aligned_faces = false; for (size_t i = 0; i < arguments.size(); ++i) { if (arguments[i].compare("-simalign") == 0) { - output_aligned_faces = true; + this->output_aligned_faces = true; output_set = true; } else if (arguments[i].compare("-hogalign") == 0) { - output_hog = true; + this->output_hog = true; output_set = true; } else if (arguments[i].compare("-2Dfp") == 0) { - output_2D_landmarks = true; + this->output_2D_landmarks = true; output_set = true; } else if (arguments[i].compare("-3Dfp") == 0) { - output_3D_landmarks = true; + this->output_3D_landmarks = true; output_set = true; } else if (arguments[i].compare("-pdmparams") == 0) { - output_model_params = true; + this->output_model_params = true; output_set = true; } else if (arguments[i].compare("-pose") == 0) { - output_pose = true; + this->output_pose = true; output_set = true; } else if (arguments[i].compare("-aus") == 0) { - output_AUs = true; + this->output_AUs = true; output_set = true; } else if (arguments[i].compare("-gaze") == 0) { - output_gaze = true; + this->output_gaze = true; output_set = true; } else if (arguments[i].compare("-tracked") == 0) { - output_tracked = true; + this->output_tracked = true; output_set = true; } } @@ -123,15 +123,15 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector if (!output_set) { - output_2D_landmarks = true; - output_3D_landmarks = true; - output_model_params = true; - output_pose = true; - output_AUs = true; - output_gaze = true; - output_hog = true; - output_tracked = true; - output_aligned_faces = true; + this->output_2D_landmarks = true; + this->output_3D_landmarks = true; + this->output_model_params = true; + this->output_pose = true; + this->output_AUs = true; + this->output_gaze = true; + this->output_hog = true; + this->output_tracked = true; + this->output_aligned_faces = true; } } diff --git a/matlab_runners/Demos/run_demo_video_multi.m b/matlab_runners/Demos/run_demo_video_multi.m index f64c248..7c7b0fa 100644 --- a/matlab_runners/Demos/run_demo_video_multi.m +++ b/matlab_runners/Demos/run_demo_video_multi.m @@ -19,7 +19,7 @@ model = 'model/main_clnf_general.txt'; % Trained on in the wild and multi-pie da %model = 'model/main_clm_wild.txt'; % Trained on in-the-wild % Create a command that will run the tracker on set of videos and display the output -command = sprintf('%s -mloc "%s" ', executable, model); +command = sprintf('%s -mloc "%s" -verbose ', executable, model); % add all videos to single argument list (so as not to load the model anew % for every video)