From fa75910b6c200a06c7fb03e6460e72850c7d9667 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 13 Nov 2017 17:48:17 +0000 Subject: [PATCH] Updating multi-face setup with the new interface. --- exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 3 + .../FaceLandmarkVidMulti.cpp | 242 +++++------------- .../FaceLandmarkVidMulti.vcxproj | 8 +- 3 files changed, 69 insertions(+), 184 deletions(-) diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index f4985fd..241f136 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -180,6 +180,9 @@ int main (int argc, char **argv) return(0); } + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); + } // Reset the model, for the next video diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 0845ced..4ed037c 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -36,6 +36,10 @@ // FaceTrackingVidMulti.cpp : Defines the entry point for the multiple face tracking console application. #include "LandmarkCoreIncludes.h" +#include "VisualizationUtils.h" +#include "Visualizer.h" +#include "SequenceCapture.h" + #include #include @@ -106,17 +110,8 @@ int main (int argc, char **argv) vector arguments = get_arguments(argc, argv); - // Some initial parameters that can be overriden from command line - vector files, tracked_videos_output, dummy_out; - - // By default try webcam 0 - int device = 0; - - // cx and cy aren't necessarilly in the image center, so need to be able to override it (start with unit vals and init them if none specified) - float fx = 600, fy = 600, cx = 0, cy = 0; LandmarkDetector::FaceModelParameters det_params(arguments); - det_params.use_face_template = true; // This is so that the model would not try re-initialising itself det_params.reinit_video_every = -1; @@ -124,113 +119,61 @@ int main (int argc, char **argv) vector det_parameters; det_parameters.push_back(det_params); - - // Get the input output file parameters - string output_codec; - LandmarkDetector::get_video_input_output_params(files, dummy_out, tracked_videos_output, output_codec, arguments); - // Get camera parameters - LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments); // The modules that are being used for tracking - vector clnf_models; + vector face_models; vector active_models; int num_faces_max = 4; - LandmarkDetector::CLNF clnf_model(det_parameters[0].model_location); - clnf_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); - clnf_model.face_detector_location = det_parameters[0].face_detector_location; + LandmarkDetector::CLNF face_model(det_parameters[0].model_location); + face_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); + face_model.face_detector_location = det_parameters[0].face_detector_location; - clnf_models.reserve(num_faces_max); + face_models.reserve(num_faces_max); - clnf_models.push_back(clnf_model); + face_models.push_back(face_model); active_models.push_back(false); for (int i = 1; i < num_faces_max; ++i) { - clnf_models.push_back(clnf_model); + face_models.push_back(face_model); active_models.push_back(false); det_parameters.push_back(det_params); } - // If multiple video files are tracked, use this to indicate if we are done - bool done = false; - int f_n = -1; + // Open a sequence + Utilities::SequenceCapture sequence_reader; - // If cx (optical axis centre) is undefined will use the image size/2 as an estimate - bool cx_undefined = false; - if(cx == 0 || cy == 0) + // A utility for visualizing the results + Utilities::Visualizer visualizer(arguments); + + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + + while(true) // this is not a for loop as we might also be reading from a webcam { - cx_undefined = true; - } - - while(!done) // this is not a for loop as we might also be reading from a webcam - { - - string current_file; - // We might specify multiple video files as arguments - if(files.size() > 0) + // The sequence reader chooses what to open based on command line arguments provided + if (!sequence_reader.Open(arguments) && sequence_reader.no_input_specified) { - f_n++; - current_file = files[f_n]; - } - - // Do some grabbing - cv::VideoCapture video_capture; - if( current_file.size() > 0 ) - { - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); + // If that fails, revert to webcam + INFO_STREAM("No input specified, attempting to open a webcam 0"); + if (!sequence_reader.OpenWebcam(0)) + ERROR_STREAM("Failed to open the webcam"); } else { - INFO_STREAM( "Attempting to capture from device: " << device ); - video_capture = cv::VideoCapture( device ); - - // Read a first frame often empty in camera - cv::Mat captured_image; - video_capture >> captured_image; + ERROR_STREAM("Failed to open a sequence"); + break; } - if (!video_capture.isOpened()) - { - FATAL_STREAM("Failed to open video source"); - return 1; - } - else INFO_STREAM( "Device or file opened"); + INFO_STREAM("Device or file opened"); - cv::Mat captured_image; - video_capture >> captured_image; - + cv::Mat captured_image = sequence_reader.GetNextFrame(); - // If optical centers are not defined just use center of image - if(cx_undefined) - { - cx = captured_image.cols / 2.0f; - cy = captured_image.rows / 2.0f; - } - int frame_count = 0; - - // saving the videos - cv::VideoWriter writerFace; - if(!tracked_videos_output.empty()) - { - try - { - writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0],output_codec[1],output_codec[2],output_codec[3]), 30, captured_image.size(), true); - } - catch(cv::Exception e) - { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - } - - // For measuring the timings - int64 t1,t0 = cv::getTickCount(); - double fps = 10; - INFO_STREAM( "Starting tracking"); while(!captured_image.empty()) @@ -253,7 +196,7 @@ int main (int argc, char **argv) vector > face_detections; bool all_models_active = true; - for(unsigned int model = 0; model < clnf_models.size(); ++model) + for(unsigned int model = 0; model < face_models.size(); ++model) { if(!active_models[model]) { @@ -267,33 +210,32 @@ int main (int argc, char **argv) if(det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) { vector confidences; - LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, clnf_models[0].face_detector_HOG, confidences); + LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, face_models[0].face_detector_HOG, confidences); } else { - LandmarkDetector::DetectFaces(face_detections, grayscale_image, clnf_models[0].face_detector_HAAR); + LandmarkDetector::DetectFaces(face_detections, grayscale_image, face_models[0].face_detector_HAAR); } } // Keep only non overlapping detections (also convert to a concurrent vector - NonOverlapingDetections(clnf_models, face_detections); + NonOverlapingDetections(face_models, face_detections); vector > face_detections_used(face_detections.size()); // Go through every model and update the tracking - tbb::parallel_for(0, (int)clnf_models.size(), [&](int model){ + tbb::parallel_for(0, (int)face_models.size(), [&](int model){ //for(unsigned int model = 0; model < clnf_models.size(); ++model) //{ bool detection_success = false; // If the current model has failed more than 4 times in a row, remove it - if(clnf_models[model].failures_in_a_row > 4) + if(face_models[model].failures_in_a_row > 4) { active_models[model] = false; - clnf_models[model].Reset(); - + face_models[model].Reset(); } // If the model is inactive reactivate it with new detections @@ -307,11 +249,11 @@ int main (int argc, char **argv) { // Reinitialise the model - clnf_models[model].Reset(); + face_models[model].Reset(); // This ensures that a wider window is used for the initial landmark localisation - clnf_models[model].detection_success = false; - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], clnf_models[model], det_parameters[model]); + face_models[model].detection_success = false; + detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], face_models[model], det_parameters[model]); // This activates the model active_models[model] = true; @@ -325,96 +267,37 @@ int main (int argc, char **argv) else { // The actual facial landmark detection / tracking - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_models[model], det_parameters[model]); + detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_models[model], det_parameters[model]); } }); + // Keeping track of FPS + fps_tracker.AddFrame(); + + visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + // Go through every model and visualise the results - for(size_t model = 0; model < clnf_models.size(); ++model) + for(size_t model = 0; model < face_models.size(); ++model) { // Visualising the results - // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised - double detection_certainty = clnf_models[model].detection_certainty; - - double visualisation_boundary = -0.1; - - // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if(detection_certainty < visualisation_boundary) + if(active_models[model]) { - LandmarkDetector::Draw(disp_image, clnf_models[model]); - - if(detection_certainty > 1) - detection_certainty = 1; - if(detection_certainty < -1) - detection_certainty = -1; - - detection_certainty = (detection_certainty + 1)/(visualisation_boundary +1); - - // A rough heuristic for box around the face width - int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - - // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate = LandmarkDetector::GetPose(clnf_models[model], fx, fy, cx, cy); - - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(disp_image, pose_estimate, cv::Scalar((1-detection_certainty)*255.0,0, detection_certainty*255), thickness, fx, fy, cx, cy); + visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty, face_models[model].detection_success); + visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty); + visualizer.SetFps(fps_tracker.GetFPS()); + visualizer.ShowObservation(); } } - - // Work out the framerate - if(frame_count % 10 == 0) - { - t1 = cv::getTickCount(); - fps = 10.0 / (double(t1-t0)/cv::getTickFrequency()); - t0 = t1; - } - - // Write out the framerate on the image before displaying it - char fpsC[255]; - sprintf(fpsC, "%d", (int)fps); - string fpsSt("FPS:"); - fpsSt += fpsC; - cv::putText(disp_image, fpsSt, cv::Point(10,20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255,0,0), 1, CV_AA); - - int num_active_models = 0; - - for( size_t active_model = 0; active_model < active_models.size(); active_model++) - { - if(active_models[active_model]) - { - num_active_models++; - } - } - - char active_m_C[255]; - sprintf(active_m_C, "%d", num_active_models); - string active_models_st("Active models:"); - active_models_st += active_m_C; - cv::putText(disp_image, active_models_st, cv::Point(10,60), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255,0,0), 1, CV_AA); - - if(!det_parameters[0].quiet_mode) - { - cv::namedWindow("tracking_result",1); - cv::imshow("tracking_result", disp_image); - } - - // output the tracked video - if(!tracked_videos_output.empty()) - { - writerFace << disp_image; - } - - video_capture >> captured_image; - + // detect key presses char character_press = cv::waitKey(1); // restart the trackers if(character_press == 'r') { - for(size_t i=0; i < clnf_models.size(); ++i) + for(size_t i=0; i < face_models.size(); ++i) { - clnf_models[i].Reset(); + face_models[i].Reset(); active_models[i] = false; } } @@ -426,22 +309,21 @@ int main (int argc, char **argv) // Update the frame count frame_count++; + + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); + } frame_count = 0; // Reset the model, for the next video - for(size_t model=0; model < clnf_models.size(); ++model) + for(size_t model=0; model < face_models.size(); ++model) { - clnf_models[model].Reset(); + face_models[model].Reset(); active_models[model] = false; } - // break out of the loop if done with all the files - if(f_n == files.size() -1) - { - done = true; - } } return 0; diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj index e8a84c6..66bfee3 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj @@ -105,7 +105,7 @@ Level3 Disabled - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -117,7 +117,7 @@ Level3 Disabled - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true @@ -131,7 +131,7 @@ MaxSpeed true true - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 MultiThreadedDLL true @@ -148,7 +148,7 @@ MaxSpeed true true - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions MultiThreadedDLL true