r
/
sustaining_gazes
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Integrating SequenceCapture to FaceLandmarkVid

This commit is contained in:
Tadas Baltrusaitis 2017-11-10 18:49:30 +00:00
parent e58ccaa73d
commit d5a3af7ece
4 changed files with 64 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
exe/FaceLandmarkVid/FaceLandmarkVid.cpp
View File

@ -46,6 +46,8 @@
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <SequenceCapture.h>
// Boost includes
#include <filesystem.hpp>
#include <filesystem/fstream.hpp>
@ -85,29 +87,29 @@ vector<string> get_arguments(int argc, char **argv)
// Some globals for tracking timing information for visualisation
double fps_tracker = -1.0;
int64 t0 = 0;
int frame_count = 0;
// Visualising the results
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy)
// Visualising the results, TODO move to separate
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy)
{
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
double detection_certainty = face_model.detection_certainty;
bool detection_success = face_model.detection_success;
double visualisation_boundary = 0.2;
double visualisation_boundary = 0.4;
// Only draw if the reliability is reasonable, the value is slightly ad-hoc
if (detection_certainty < visualisation_boundary)
if (detection_certainty > visualisation_boundary)
{
LandmarkDetector::Draw(captured_image, face_model);
double vis_certainty = detection_certainty;
if (vis_certainty > 1)
vis_certainty = 1;
if (vis_certainty < -1)
vis_certainty = -1;
vis_certainty = (vis_certainty + 1) / (visualisation_boundary + 1);
// Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking
vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary);
// A rough heuristic for box around the face width
int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0);
@ -115,15 +117,15 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy);
// Draw it in reddish if uncertain, blueish if certain
LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy);
LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar(vis_certainty*255.0, 0, (1 - vis_certainty) * 255), thickness, fx, fy, cx, cy);
if (det_parameters.track_gaze && detection_success && face_model.eye_model)
{
GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy);
}
}
// Work out the framerate
// Work out the framerate TODO
if (frame_count % 10 == 0)
{
double t1 = cv::getTickCount();
@ -136,13 +138,9 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
std::sprintf(fpsC, "%d", (int)fps_tracker);
string fpsSt("FPS:");
fpsSt += fpsC;
cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0));
cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
frame_count++;
if (!det_parameters.quiet_mode)
{
cv::namedWindow("tracking_result", 1);
cv::imshow("tracking_result", captured_image);
}
}
int main (int argc, char **argv)
@ -150,152 +148,51 @@ int main (int argc, char **argv)
vector<string> arguments = get_arguments(argc, argv);
// Some initial parameters that can be overriden from command line
vector<string> files, output_video_files, out_dummy;
// By default try webcam 0
int device = 0;
LandmarkDetector::FaceModelParameters det_parameters(arguments);
det_parameters.track_gaze = true;
// Get the input output file parameters
// Indicates that rotation should be with respect to world or camera coordinates
string output_codec;
LandmarkDetector::get_video_input_output_params(files, out_dummy, output_video_files, output_codec, arguments);
// The modules that are being used for tracking
LandmarkDetector::CLNF clnf_model(det_parameters.model_location);
// Grab camera parameters, if they are not defined (approximate values will be used)
float fx = 0, fy = 0, cx = 0, cy = 0;
// Get camera parameters
LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments);
// Open a sequence
Utilities::SequenceCapture sequence_reader;
// If cx (optical axis centre) is undefined will use the image size/2 as an estimate
bool cx_undefined = false;
bool fx_undefined = false;
if (cx == 0 || cy == 0)
while (true) // this is not a for loop as we might also be reading from a webcam
{
cx_undefined = true;
}
if (fx == 0 || fy == 0)
{
fx_undefined = true;
}
// If multiple video files are tracked, use this to indicate if we are done
bool done = false;
int f_n = -1;
det_parameters.track_gaze = true;
// The sequence reader chooses what to open based on command line arguments provided
bool reader_opened = sequence_reader.Open(arguments);
while(!done) // this is not a for loop as we might also be reading from a webcam
{
string current_file;
// We might specify multiple video files as arguments
if(files.size() > 0)
if(!reader_opened && sequence_reader.no_input_specified)
{
f_n++;
current_file = files[f_n];
// If that fails, revert to webcam
INFO_STREAM("No input specified, attempting to open a webcam 0");
if (!sequence_reader.OpenWebcam(0))
ERROR_STREAM("Failed to open the webcam");
}
else
{
// If we want to write out from webcam
f_n = 0;
}
// Do some grabbing
cv::VideoCapture video_capture;
if( current_file.size() > 0 )
{
if (!boost::filesystem::exists(current_file))
{
FATAL_STREAM("File does not exist");
return 1;
}
current_file = boost::filesystem::path(current_file).generic_string();
INFO_STREAM( "Attempting to read from file: " << current_file );
video_capture = cv::VideoCapture( current_file );
}
else
{
INFO_STREAM( "Attempting to capture from device: " << device );
video_capture = cv::VideoCapture( device );
// Read a first frame often empty in camera
cv::Mat captured_image;
video_capture >> captured_image;
ERROR_STREAM("Failed to open a sequence");
break;
}
if (!video_capture.isOpened())
{
FATAL_STREAM("Failed to open video source");
return 1;
}
else INFO_STREAM( "Device or file opened");
INFO_STREAM("Device or file opened");
cv::Mat captured_image;
video_capture >> captured_image;
// If optical centers are not defined just use center of image
if (cx_undefined)
cv::Mat captured_image;
captured_image = sequence_reader.GetNextFrame();
INFO_STREAM("Starting tracking");
while (!captured_image.empty()) // this is not a for loop as we might also be reading from a webcam
{
cx = captured_image.cols / 2.0f;
cy = captured_image.rows / 2.0f;
}
// Use a rough guess-timate of focal length
if (fx_undefined)
{
fx = 500 * (captured_image.cols / 640.0);
fy = 500 * (captured_image.rows / 480.0);
fx = (fx + fy) / 2.0;
fy = fx;
}
int frame_count = 0;
// saving the videos
cv::VideoWriter writerFace;
if (!output_video_files.empty())
{
try
{
writerFace = cv::VideoWriter(output_video_files[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), 30, captured_image.size(), true);
}
catch(cv::Exception e)
{
WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
}
}
// Use for timestamping if using a webcam
int64 t_initial = cv::getTickCount();
INFO_STREAM( "Starting tracking");
while(!captured_image.empty())
{
// Reading the images
cv::Mat_<uchar> grayscale_image;
if(captured_image.channels() == 3)
{
cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY);
}
else
{
grayscale_image = captured_image.clone();
}
// The actual facial landmark detection / tracking
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_model, det_parameters);
// Visualising the results
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
double detection_certainty = clnf_model.detection_certainty;
@ -306,52 +203,32 @@ int main (int argc, char **argv)
if (det_parameters.track_gaze && detection_success && clnf_model.eye_model)
{
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true);
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false);
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
}
visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy);
// output the tracked video
if (!output_video_files.empty())
{
writerFace << captured_image;
}
visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
video_capture >> captured_image;
// detect key presses
char character_press = cv::waitKey(1);
// restart the tracker
if(character_press == 'r')
if (character_press == 'r')
{
clnf_model.Reset();
}
// quit the application
else if(character_press=='q')
else if (character_press == 'q')
{
return(0);
}
// Update the frame count
frame_count++;
}
frame_count = 0;
// Reset the model, for the next video
clnf_model.Reset();
// break out of the loop if done with all the files (or using a webcam)
if(f_n == files.size() -1 || files.empty())
{
done = true;
}
}
}
return 0;
}

11
exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj
View File

@ -112,7 +112,7 @@
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
@ -128,7 +128,7 @@
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
@ -147,7 +147,7 @@
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
@ -169,7 +169,7 @@
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
@ -195,6 +195,9 @@
<ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">

4
lib/local/Utilities/include/SequenceCapture.h
View File

@ -96,6 +96,9 @@ namespace Utilities
// Name of the video file, image directory, or the webcam
std::string name;
// Allows to differentiate if failed because no input specified or if failed to open a specified input
bool no_input_specified;
private:
// Used for capturing webcam and video
@ -124,6 +127,7 @@ namespace Utilities
void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
};
}
#endif

13
lib/local/Utilities/src/SequenceCapture.cpp
View File

@ -154,6 +154,8 @@ bool SequenceCapture::Open(std::vector<std::string> arguments)
}
}
no_input_specified = !file_found;
// Based on what was read in open the sequence TODO
if (device != -1)
{
@ -167,7 +169,10 @@ bool SequenceCapture::Open(std::vector<std::string> arguments)
{
return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy);
}
// If none opened, return false
// If no input found return false and set a flag for it
no_input_specified = true;
return false;
}
@ -175,6 +180,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height,
{
INFO_STREAM("Attempting to read from webcam: " << device);
no_input_specified = false;
if (device < 0)
{
std::cout << "Specify a valid device" << std::endl;
@ -232,6 +239,8 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy,
{
INFO_STREAM("Attempting to read from file: " << video_file);
no_input_specified = false;
latest_frame = cv::Mat();
latest_gray_frame = cv::Mat();
@ -272,6 +281,8 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f
{
INFO_STREAM("Attempting to read from directory: " << directory);
no_input_specified = false;
image_files.clear();
boost::filesystem::path image_directory(directory);