diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 48eb686..0891545 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -4,7 +4,7 @@ // // ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. // IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. // // License can be found in OpenFace-license.txt @@ -14,21 +14,21 @@ // reports and manuals, must cite at least one of the following works: // // OpenFace: an open source facial behavior analysis toolkit -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency -// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// Tadas Baltru�aitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 // // Rendering of Eyes for Eye-Shape Registration and Gaze Estimation -// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling -// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// Erroll Wood, Tadas Baltru�aitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 // // Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection -// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson -// in Facial Expression Recognition and Analysis Challenge, -// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// Tadas Baltru�aitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 // // Constrained Local Neural Fields for robust facial landmark detection in the wild. -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. -// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// Tadas Baltru�aitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// @@ -94,7 +94,7 @@ void NonOverlapingDetections(const vector& clnf_models, // See if the detections intersect cv::Rect_ model_rect = clnf_models[model].GetBoundingBox(); - + for(int detection = face_detections.size()-1; detection >=0; --detection) { double intersection_area = (model_rect & face_detections[detection]).area(); @@ -130,17 +130,17 @@ int main (int argc, char **argv) vector det_parameters; det_parameters.push_back(det_params); - + // The modules that are being used for tracking vector face_models; vector active_models; - int num_faces_max = 4; + int num_faces_max = 15; LandmarkDetector::CLNF face_model(det_parameters[0].model_location); face_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); face_model.face_detector_location = det_parameters[0].face_detector_location; - + face_models.reserve(num_faces_max); face_models.push_back(face_model); @@ -152,7 +152,7 @@ int main (int argc, char **argv) active_models.push_back(false); det_parameters.push_back(det_params); } - + // Load facial feature extractor and AU analyser (make sure it is static, as we don't reidentify faces) FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); face_analysis_params.OptimizeForImages(); @@ -195,10 +195,16 @@ int main (int argc, char **argv) Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); + // for some reason not accepted as cli parameter, as we don't need it: disable it anyway + recording_params.setOutputAUs(false); + recording_params.setOutputHOG(false); + recording_params.setOutputAlignedFaces(false); + recording_params.setOutputTracked(false); if (!face_model.eye_model) { recording_params.setOutputGaze(false); - } + } + Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); @@ -217,11 +223,11 @@ int main (int argc, char **argv) INFO_STREAM( "Starting tracking"); while(!captured_image.empty()) - { + { // Reading the images cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); - + vector > face_detections; bool all_models_active = true; @@ -232,10 +238,10 @@ int main (int argc, char **argv) all_models_active = false; } } - + // Get the detections (every 8th frame and when there are free models available for tracking) if(frame_count % 8 == 0 && !all_models_active) - { + { if(det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) { vector confidences; @@ -262,7 +268,7 @@ int main (int argc, char **argv) // If the current model has failed more than 4 times in a row, remove it if(face_models[model].failures_in_a_row > 4) - { + { active_models[model] = false; face_models[model].Reset(); } @@ -270,13 +276,13 @@ int main (int argc, char **argv) // If the model is inactive reactivate it with new detections if(!active_models[model]) { - + for(size_t detection_ind = 0; detection_ind < face_detections.size(); ++detection_ind) { // if it was not taken by another tracker take it (if it is false swap it to true and enter detection, this makes it parallel safe) if(face_detections_used[detection_ind].compare_and_swap(true, false) == false) { - + // Reinitialise the model face_models[model].Reset(); @@ -299,12 +305,15 @@ int main (int argc, char **argv) detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_models[model], det_parameters[model]); } }); - + // Keeping track of FPS fps_tracker.AddFrame(); visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + std::stringstream jsonOutput; + jsonOutput << "["; + int jsonFaceId = 0; // Go through every model and detect eye gaze, record results and visualise the results for(size_t model = 0; model < face_models.size(); ++model) { @@ -312,7 +321,7 @@ int main (int argc, char **argv) if(active_models[model]) { - // Estimate head pose and eye gaze + // Estimate head pose and eye gaze cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); cv::Point3f gaze_direction0(0, 0, 0); cv::Point3f gaze_direction1(0, 0, 0); cv::Vec2d gaze_angle(0, 0); @@ -337,14 +346,30 @@ int main (int argc, char **argv) face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } + cv::Vec6d head_pose = LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + // Visualize the features visualizer.SetObservationFaceAlign(sim_warped_img); visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty); - visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty); + visualizer.SetObservationPose(head_pose, face_models[model].detection_certainty); visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty); visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + if(face_models[model].detection_success && face_model.eye_model) { + if(jsonFaceId > 0){ + jsonOutput << ","; + } + jsonFaceId++; + jsonOutput << "{\"fid\":"; + jsonOutput << model << ", \"confidence\":" << face_models[model].detection_certainty; + // gaze_angle_x, gaze_angle_y Eye gaze direction in radians in world coordinates averaged for both eyes and converted into more easy to use format than gaze vectors. If a person is looking left-right this will results in the change of gaze_angle_x and, if a person is looking up-down this will result in change of gaze_angle_y, if a person is looking straight ahead both of the angles will be close to 0 (within measurement error) + jsonOutput << ", \"gaze_angle\": [" << gaze_angle[0] << ", " << gaze_angle[1] << "]"; + jsonOutput << ", head_pos: [" << head_pose[0] << ", " << head_pose[1] << ", " << head_pose[2] << "]"; + jsonOutput << ", head_rot: [" << head_pose[3] << ", " << head_pose[4] << ", " << head_pose[5] << "]"; + jsonOutput << "}"; + } + // Output features open_face_rec.SetObservationHOG(face_models[model].detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); @@ -364,9 +389,12 @@ int main (int argc, char **argv) } visualizer.SetFps(fps_tracker.GetFPS()); + jsonOutput << "]"; + std::cout << jsonOutput.str() << std::endl; + // show visualization and detect key presses char character_press = visualizer.ShowObservation(); - + // restart the trackers if(character_press == 'r') { @@ -389,7 +417,7 @@ int main (int argc, char **argv) captured_image = sequence_reader.GetNextFrame(); } - + frame_count = 0; // Reset the model, for the next video @@ -405,4 +433,3 @@ int main (int argc, char **argv) return 0; } - diff --git a/lib/local/Utilities/include/RecorderOpenFaceParameters.h b/lib/local/Utilities/include/RecorderOpenFaceParameters.h index 74efa66..1a3b5fa 100644 --- a/lib/local/Utilities/include/RecorderOpenFaceParameters.h +++ b/lib/local/Utilities/include/RecorderOpenFaceParameters.h @@ -3,7 +3,7 @@ // // ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. // IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. // // License can be found in OpenFace-license.txt @@ -13,21 +13,21 @@ // reports and manuals, must cite at least one of the following works: // // OpenFace: an open source facial behavior analysis toolkit -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency -// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// Tadas Baltru�aitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 // // Rendering of Eyes for Eye-Shape Registration and Gaze Estimation -// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling -// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// Erroll Wood, Tadas Baltru�aitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 // // Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection -// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson -// in Facial Expression Recognition and Analysis Challenge, -// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// Tadas Baltru�aitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 // // Constrained Local Neural Fields for robust facial landmark detection in the wild. -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. -// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// Tadas Baltru�aitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// @@ -79,9 +79,12 @@ namespace Utilities void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; } void setOutputGaze(bool output_gaze) { this->output_gaze = output_gaze; } + void setOutputHOG(bool output_HOG) { this->output_hog = output_HOG; } + void setOutputAlignedFaces(bool output_aligned_faces) { this->output_aligned_faces = output_aligned_faces; } + void setOutputTracked(bool output_tracked) { this->output_tracked = output_tracked; } private: - + // If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face bool is_sequence; // If the data is coming from a webcam @@ -97,7 +100,7 @@ namespace Utilities bool output_hog; bool output_tracked; bool output_aligned_faces; - + // Some video recording parameters std::string output_codec; double fps_vid_out; diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 19d88dc..964da93 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -3,7 +3,7 @@ // // ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. // IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. // // License can be found in OpenFace-license.txt @@ -13,21 +13,21 @@ // reports and manuals, must cite at least one of the following works: // // OpenFace: an open source facial behavior analysis toolkit -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency -// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// Tadas Baltru�aitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 // // Rendering of Eyes for Eye-Shape Registration and Gaze Estimation -// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling -// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// Erroll Wood, Tadas Baltru�aitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 // // Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection -// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson -// in Facial Expression Recognition and Analysis Challenge, -// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// Tadas Baltru�aitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 // // Constrained Local Neural Fields for robust facial landmark detection in the wild. -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. -// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// Tadas Baltru�aitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// @@ -126,7 +126,7 @@ void RecorderOpenFace::PrepareRecording(const std::string& in_filename) hog_recorder.Open(hog_filename); } - // saving the videos + // saving the videos if (params.outputTracked()) { if (params.isSequence()) @@ -275,8 +275,7 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) void RecorderOpenFace::WriteObservation() { - - // Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers) + // Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers) if(!csv_recorder.isOpen()) { // As we are writing out the header, work out some things like number of landmarks, names of AUs etc. @@ -312,8 +311,34 @@ void RecorderOpenFace::WriteObservation() csv_recorder.Open(csv_filename, params.isSequence(), params.output2DLandmarks(), params.output3DLandmarks(), params.outputPDMParams(), params.outputPose(), params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg); } + // + // if(landmark_detection_success){ + // + // std::cout << "{\"fid\":"; + // std::cout << face_id << ", \"confidence\":" << landmark_detection_confidence; + // // + // // std::cout << ", \"gaze_directions\": [[" << gaze_direction0.x << ", " << gaze_direction0.y << ", " << gaze_direction0.z << "]" + // // << ", [" << gaze_direction1.x << ", " << gaze_direction1.y << ", " << gaze_direction1.z << "]]"; + // + // // gaze_angle_x, gaze_angle_y Eye gaze direction in radians in world coordinates averaged for both eyes and converted into more easy to use format than gaze vectors. If a person is looking left-right this will results in the change of gaze_angle_x and, if a person is looking up-down this will result in change of gaze_angle_y, if a person is looking straight ahead both of the angles will be close to 0 (within measurement error) + // std::cout << ", \"gaze_angle\": [" << gaze_angle[0] << ", " << gaze_angle[1] << "]"; + // // + // // std::cout << ",\"eye_lmk3d\"[[]"; + // // // Output the 3D eye landmarks + // // for (auto eye_lmk : eye_landmarks3D) + // // { + // // std::cout << ", [" << eye_lmk.x << "," << eye_lmk.y << "," << eye_lmk.z << "]"; + // // } + // // std::cout << "]"; + // + // std::cout << ", position: [" << head_pose[0] << ", " << head_pose[1] << ", " << head_pose[2] << "]"; + // // rotation: + // // std::cout << ", " << pose_estimate[3] << ", " << pose_estimate[4] << ", " << pose_estimate[5]; + // + // std::cout << "}" << std::endl; + // } - this->csv_recorder.WriteLine(face_id, frame_number, timestamp, landmark_detection_success, + this->csv_recorder.WriteLine(face_id, frame_number, timestamp, landmark_detection_success, landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose, gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences); @@ -446,6 +471,3 @@ void RecorderOpenFace::Close() metadata_file.close(); } - - -