diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 8bda263..f7c4b37 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -46,6 +46,8 @@ #include #include +#include + // Boost includes #include #include @@ -85,29 +87,29 @@ vector get_arguments(int argc, char **argv) // Some globals for tracking timing information for visualisation double fps_tracker = -1.0; int64 t0 = 0; +int frame_count = 0; -// Visualising the results -void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) +// Visualising the results, TODO move to separate +void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy) { // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised double detection_certainty = face_model.detection_certainty; bool detection_success = face_model.detection_success; - double visualisation_boundary = 0.2; + double visualisation_boundary = 0.4; // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if (detection_certainty < visualisation_boundary) + if (detection_certainty > visualisation_boundary) { LandmarkDetector::Draw(captured_image, face_model); double vis_certainty = detection_certainty; if (vis_certainty > 1) vis_certainty = 1; - if (vis_certainty < -1) - vis_certainty = -1; - vis_certainty = (vis_certainty + 1) / (visualisation_boundary + 1); + // Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking + vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary); // A rough heuristic for box around the face width int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); @@ -115,15 +117,15 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy); - + LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar(vis_certainty*255.0, 0, (1 - vis_certainty) * 255), thickness, fx, fy, cx, cy); + if (det_parameters.track_gaze && detection_success && face_model.eye_model) { GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); } } - // Work out the framerate + // Work out the framerate TODO if (frame_count % 10 == 0) { double t1 = cv::getTickCount(); @@ -136,13 +138,9 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f std::sprintf(fpsC, "%d", (int)fps_tracker); string fpsSt("FPS:"); fpsSt += fpsC; - cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0)); + cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); + frame_count++; - if (!det_parameters.quiet_mode) - { - cv::namedWindow("tracking_result", 1); - cv::imshow("tracking_result", captured_image); - } } int main (int argc, char **argv) @@ -150,152 +148,51 @@ int main (int argc, char **argv) vector arguments = get_arguments(argc, argv); - // Some initial parameters that can be overriden from command line - vector files, output_video_files, out_dummy; - - // By default try webcam 0 - int device = 0; - LandmarkDetector::FaceModelParameters det_parameters(arguments); + det_parameters.track_gaze = true; - // Get the input output file parameters - - // Indicates that rotation should be with respect to world or camera coordinates - string output_codec; - LandmarkDetector::get_video_input_output_params(files, out_dummy, output_video_files, output_codec, arguments); - // The modules that are being used for tracking LandmarkDetector::CLNF clnf_model(det_parameters.model_location); - // Grab camera parameters, if they are not defined (approximate values will be used) - float fx = 0, fy = 0, cx = 0, cy = 0; - // Get camera parameters - LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments); + // Open a sequence + Utilities::SequenceCapture sequence_reader; - // If cx (optical axis centre) is undefined will use the image size/2 as an estimate - bool cx_undefined = false; - bool fx_undefined = false; - if (cx == 0 || cy == 0) + while (true) // this is not a for loop as we might also be reading from a webcam { - cx_undefined = true; - } - if (fx == 0 || fy == 0) - { - fx_undefined = true; - } - // If multiple video files are tracked, use this to indicate if we are done - bool done = false; - int f_n = -1; - - det_parameters.track_gaze = true; + // The sequence reader chooses what to open based on command line arguments provided + bool reader_opened = sequence_reader.Open(arguments); - while(!done) // this is not a for loop as we might also be reading from a webcam - { - - string current_file; - - // We might specify multiple video files as arguments - if(files.size() > 0) + if(!reader_opened && sequence_reader.no_input_specified) { - f_n++; - current_file = files[f_n]; + // If that fails, revert to webcam + INFO_STREAM("No input specified, attempting to open a webcam 0"); + if (!sequence_reader.OpenWebcam(0)) + ERROR_STREAM("Failed to open the webcam"); } else { - // If we want to write out from webcam - f_n = 0; - } - - // Do some grabbing - cv::VideoCapture video_capture; - if( current_file.size() > 0 ) - { - if (!boost::filesystem::exists(current_file)) - { - FATAL_STREAM("File does not exist"); - return 1; - } - - current_file = boost::filesystem::path(current_file).generic_string(); - - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); - } - else - { - INFO_STREAM( "Attempting to capture from device: " << device ); - video_capture = cv::VideoCapture( device ); - - // Read a first frame often empty in camera - cv::Mat captured_image; - video_capture >> captured_image; + ERROR_STREAM("Failed to open a sequence"); + break; } - if (!video_capture.isOpened()) - { - FATAL_STREAM("Failed to open video source"); - return 1; - } - else INFO_STREAM( "Device or file opened"); + INFO_STREAM("Device or file opened"); cv::Mat captured_image; - video_capture >> captured_image; - // If optical centers are not defined just use center of image - if (cx_undefined) + cv::Mat captured_image; + captured_image = sequence_reader.GetNextFrame(); + + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) // this is not a for loop as we might also be reading from a webcam { - cx = captured_image.cols / 2.0f; - cy = captured_image.rows / 2.0f; - } - // Use a rough guess-timate of focal length - if (fx_undefined) - { - fx = 500 * (captured_image.cols / 640.0); - fy = 500 * (captured_image.rows / 480.0); - - fx = (fx + fy) / 2.0; - fy = fx; - } - - int frame_count = 0; - - // saving the videos - cv::VideoWriter writerFace; - if (!output_video_files.empty()) - { - try - { - writerFace = cv::VideoWriter(output_video_files[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), 30, captured_image.size(), true); - } - catch(cv::Exception e) - { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - } - - // Use for timestamping if using a webcam - int64 t_initial = cv::getTickCount(); - - INFO_STREAM( "Starting tracking"); - while(!captured_image.empty()) - { // Reading the images cv::Mat_ grayscale_image; - if(captured_image.channels() == 3) - { - cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); - } - else - { - grayscale_image = captured_image.clone(); - } - // The actual facial landmark detection / tracking bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_model, det_parameters); - + // Visualising the results // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised double detection_certainty = clnf_model.detection_certainty; @@ -306,52 +203,32 @@ int main (int argc, char **argv) if (det_parameters.track_gaze && detection_success && clnf_model.eye_model) { - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); } - visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); - - // output the tracked video - if (!output_video_files.empty()) - { - writerFace << captured_image; - } + visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - - video_capture >> captured_image; - // detect key presses char character_press = cv::waitKey(1); - + // restart the tracker - if(character_press == 'r') + if (character_press == 'r') { clnf_model.Reset(); } // quit the application - else if(character_press=='q') + else if (character_press == 'q') { return(0); } - // Update the frame count - frame_count++; - } - frame_count = 0; - // Reset the model, for the next video clnf_model.Reset(); - - // break out of the loop if done with all the files (or using a webcam) - if(f_n == files.size() -1 || files.empty()) - { - done = true; - } - } + } return 0; } diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj index 858265c..480dc6e 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj @@ -112,7 +112,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -128,7 +128,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -147,7 +147,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed StreamingSIMDExtensions2 @@ -169,7 +169,7 @@ true WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed AdvancedVectorExtensions @@ -195,6 +195,9 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index 18f90cf..1b1fcba 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -96,6 +96,9 @@ namespace Utilities // Name of the video file, image directory, or the webcam std::string name; + // Allows to differentiate if failed because no input specified or if failed to open a specified input + bool no_input_specified; + private: // Used for capturing webcam and video @@ -124,6 +127,7 @@ namespace Utilities void SetCameraIntrinsics(float fx, float fy, float cx, float cy); + }; } #endif \ No newline at end of file diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index 46fc6a2..b368245 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -154,6 +154,8 @@ bool SequenceCapture::Open(std::vector arguments) } } + no_input_specified = !file_found; + // Based on what was read in open the sequence TODO if (device != -1) { @@ -167,7 +169,10 @@ bool SequenceCapture::Open(std::vector arguments) { return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy); } - // If none opened, return false + + // If no input found return false and set a flag for it + no_input_specified = true; + return false; } @@ -175,6 +180,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, { INFO_STREAM("Attempting to read from webcam: " << device); + no_input_specified = false; + if (device < 0) { std::cout << "Specify a valid device" << std::endl; @@ -232,6 +239,8 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, { INFO_STREAM("Attempting to read from file: " << video_file); + no_input_specified = false; + latest_frame = cv::Mat(); latest_gray_frame = cv::Mat(); @@ -272,6 +281,8 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f { INFO_STREAM("Attempting to read from directory: " << directory); + no_input_specified = false; + image_files.clear(); boost::filesystem::path image_directory(directory);