r
/
sustaining_gazes
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Integrating SequenceCapture to FaceLandmarkVid

This commit is contained in:
Tadas Baltrusaitis 2017-11-10 18:49:30 +00:00
parent e58ccaa73d
commit d5a3af7ece
4 changed files with 64 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
exe/FaceLandmarkVid/FaceLandmarkVid.cpp
View File

@ -46,6 +46,8 @@
#include <opencv2/imgproc.hpp> #include <opencv2/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp> #include <opencv2/highgui/highgui.hpp>
#include <SequenceCapture.h>
// Boost includes // Boost includes
#include <filesystem.hpp> #include <filesystem.hpp>
#include <filesystem/fstream.hpp> #include <filesystem/fstream.hpp>
@ -85,29 +87,29 @@ vector<string> get_arguments(int argc, char **argv)
// Some globals for tracking timing information for visualisation // Some globals for tracking timing information for visualisation
double fps_tracker = -1.0; double fps_tracker = -1.0;
int64 t0 = 0; int64 t0 = 0;
int frame_count = 0;
// Visualising the results // Visualising the results, TODO move to separate
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy)
{ {
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
double detection_certainty = face_model.detection_certainty; double detection_certainty = face_model.detection_certainty;
bool detection_success = face_model.detection_success; bool detection_success = face_model.detection_success;
double visualisation_boundary = 0.2; double visualisation_boundary = 0.4;
// Only draw if the reliability is reasonable, the value is slightly ad-hoc // Only draw if the reliability is reasonable, the value is slightly ad-hoc
if (detection_certainty < visualisation_boundary) if (detection_certainty > visualisation_boundary)
{ {
LandmarkDetector::Draw(captured_image, face_model); LandmarkDetector::Draw(captured_image, face_model);
double vis_certainty = detection_certainty; double vis_certainty = detection_certainty;
if (vis_certainty > 1) if (vis_certainty > 1)
vis_certainty = 1; vis_certainty = 1;
if (vis_certainty < -1)
vis_certainty = -1;
vis_certainty = (vis_certainty + 1) / (visualisation_boundary + 1); // Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking
vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary);
// A rough heuristic for box around the face width // A rough heuristic for box around the face width
int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0);
@ -115,15 +117,15 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy);
// Draw it in reddish if uncertain, blueish if certain // Draw it in reddish if uncertain, blueish if certain
LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy); LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar(vis_certainty*255.0, 0, (1 - vis_certainty) * 255), thickness, fx, fy, cx, cy);
if (det_parameters.track_gaze && detection_success && face_model.eye_model) if (det_parameters.track_gaze && detection_success && face_model.eye_model)
{ {
GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy);
} }
} }
// Work out the framerate // Work out the framerate TODO
if (frame_count % 10 == 0) if (frame_count % 10 == 0)
{ {
double t1 = cv::getTickCount(); double t1 = cv::getTickCount();
@ -136,13 +138,9 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
std::sprintf(fpsC, "%d", (int)fps_tracker); std::sprintf(fpsC, "%d", (int)fps_tracker);
string fpsSt("FPS:"); string fpsSt("FPS:");
fpsSt += fpsC; fpsSt += fpsC;
cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0)); cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
frame_count++;
if (!det_parameters.quiet_mode)
{
cv::namedWindow("tracking_result", 1);
cv::imshow("tracking_result", captured_image);
}
} }
int main (int argc, char **argv) int main (int argc, char **argv)
@ -150,152 +148,51 @@ int main (int argc, char **argv)
vector<string> arguments = get_arguments(argc, argv); vector<string> arguments = get_arguments(argc, argv);
// Some initial parameters that can be overriden from command line
vector<string> files, output_video_files, out_dummy;
// By default try webcam 0
int device = 0;
LandmarkDetector::FaceModelParameters det_parameters(arguments); LandmarkDetector::FaceModelParameters det_parameters(arguments);
det_parameters.track_gaze = true;
// Get the input output file parameters
// Indicates that rotation should be with respect to world or camera coordinates
string output_codec;
LandmarkDetector::get_video_input_output_params(files, out_dummy, output_video_files, output_codec, arguments);
// The modules that are being used for tracking // The modules that are being used for tracking
LandmarkDetector::CLNF clnf_model(det_parameters.model_location); LandmarkDetector::CLNF clnf_model(det_parameters.model_location);
// Grab camera parameters, if they are not defined (approximate values will be used) // Open a sequence
float fx = 0, fy = 0, cx = 0, cy = 0; Utilities::SequenceCapture sequence_reader;
// Get camera parameters
LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments);
// If cx (optical axis centre) is undefined will use the image size/2 as an estimate while (true) // this is not a for loop as we might also be reading from a webcam
bool cx_undefined = false;
bool fx_undefined = false;
if (cx == 0 || cy == 0)
{ {
cx_undefined = true;
}
if (fx == 0 || fy == 0)
{
fx_undefined = true;
}
// If multiple video files are tracked, use this to indicate if we are done // The sequence reader chooses what to open based on command line arguments provided
bool done = false; bool reader_opened = sequence_reader.Open(arguments);
int f_n = -1;
det_parameters.track_gaze = true;
while(!done) // this is not a for loop as we might also be reading from a webcam if(!reader_opened && sequence_reader.no_input_specified)
{
string current_file;
// We might specify multiple video files as arguments
if(files.size() > 0)
{ {
f_n++; // If that fails, revert to webcam
current_file = files[f_n]; INFO_STREAM("No input specified, attempting to open a webcam 0");
if (!sequence_reader.OpenWebcam(0))
ERROR_STREAM("Failed to open the webcam");
} }
else else
{ {
// If we want to write out from webcam ERROR_STREAM("Failed to open a sequence");
f_n = 0; break;
}
// Do some grabbing
cv::VideoCapture video_capture;
if( current_file.size() > 0 )
{
if (!boost::filesystem::exists(current_file))
{
FATAL_STREAM("File does not exist");
return 1;
}
current_file = boost::filesystem::path(current_file).generic_string();
INFO_STREAM( "Attempting to read from file: " << current_file );
video_capture = cv::VideoCapture( current_file );
}
else
{
INFO_STREAM( "Attempting to capture from device: " << device );
video_capture = cv::VideoCapture( device );
// Read a first frame often empty in camera
cv::Mat captured_image;
video_capture >> captured_image;
} }
if (!video_capture.isOpened()) INFO_STREAM("Device or file opened");
{
FATAL_STREAM("Failed to open video source");
return 1;
}
else INFO_STREAM( "Device or file opened");
cv::Mat captured_image; cv::Mat captured_image;
video_capture >> captured_image;
// If optical centers are not defined just use center of image cv::Mat captured_image;
if (cx_undefined) captured_image = sequence_reader.GetNextFrame();
INFO_STREAM("Starting tracking");
while (!captured_image.empty()) // this is not a for loop as we might also be reading from a webcam
{ {
cx = captured_image.cols / 2.0f;
cy = captured_image.rows / 2.0f;
}
// Use a rough guess-timate of focal length
if (fx_undefined)
{
fx = 500 * (captured_image.cols / 640.0);
fy = 500 * (captured_image.rows / 480.0);
fx = (fx + fy) / 2.0;
fy = fx;
}
int frame_count = 0;
// saving the videos
cv::VideoWriter writerFace;
if (!output_video_files.empty())
{
try
{
writerFace = cv::VideoWriter(output_video_files[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), 30, captured_image.size(), true);
}
catch(cv::Exception e)
{
WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
}
}
// Use for timestamping if using a webcam
int64 t_initial = cv::getTickCount();
INFO_STREAM( "Starting tracking");
while(!captured_image.empty())
{
// Reading the images // Reading the images
cv::Mat_<uchar> grayscale_image; cv::Mat_<uchar> grayscale_image;
if(captured_image.channels() == 3)
{
cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY);
}
else
{
grayscale_image = captured_image.clone();
}
// The actual facial landmark detection / tracking // The actual facial landmark detection / tracking
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_model, det_parameters); bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_model, det_parameters);
// Visualising the results // Visualising the results
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
double detection_certainty = clnf_model.detection_certainty; double detection_certainty = clnf_model.detection_certainty;
@ -306,52 +203,32 @@ int main (int argc, char **argv)
if (det_parameters.track_gaze && detection_success && clnf_model.eye_model) if (det_parameters.track_gaze && detection_success && clnf_model.eye_model)
{ {
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
} }
visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// output the tracked video
if (!output_video_files.empty())
{
writerFace << captured_image;
}
video_capture >> captured_image;
// detect key presses // detect key presses
char character_press = cv::waitKey(1); char character_press = cv::waitKey(1);
// restart the tracker // restart the tracker
if(character_press == 'r') if (character_press == 'r')
{ {
clnf_model.Reset(); clnf_model.Reset();
} }
// quit the application // quit the application
else if(character_press=='q') else if (character_press == 'q')
{ {
return(0); return(0);
} }
// Update the frame count
frame_count++;
} }
frame_count = 0;
// Reset the model, for the next video // Reset the model, for the next video
clnf_model.Reset(); clnf_model.Reset();
// break out of the loop if done with all the files (or using a webcam)
if(f_n == files.size() -1 || files.empty())
{
done = true;
}
}
}
return 0; return 0;
} }

11
exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj
View File

@ -112,7 +112,7 @@
<WarningLevel>Level3</WarningLevel> <WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization> <Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport> <OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation> <MultiProcessorCompilation>true</MultiProcessorCompilation>
@ -128,7 +128,7 @@
<WarningLevel>Level3</WarningLevel> <WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization> <Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport> <OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation> <MultiProcessorCompilation>true</MultiProcessorCompilation>
@ -147,7 +147,7 @@
</FunctionLevelLinking> </FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions> <IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport> <OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed> <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
@ -169,7 +169,7 @@
</FunctionLevelLinking> </FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions> <IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport> <OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed> <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet> <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
@ -195,6 +195,9 @@
<ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj"> <ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project> <Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference> </ProjectReference>
<ProjectReference Include="..\..\lib\local\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">

4
lib/local/Utilities/include/SequenceCapture.h
View File

@ -96,6 +96,9 @@ namespace Utilities
// Name of the video file, image directory, or the webcam // Name of the video file, image directory, or the webcam
std::string name; std::string name;
// Allows to differentiate if failed because no input specified or if failed to open a specified input
bool no_input_specified;
private: private:
// Used for capturing webcam and video // Used for capturing webcam and video
@ -124,6 +127,7 @@ namespace Utilities
void SetCameraIntrinsics(float fx, float fy, float cx, float cy); void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
}; };
} }
#endif #endif

13
lib/local/Utilities/src/SequenceCapture.cpp
View File

@ -154,6 +154,8 @@ bool SequenceCapture::Open(std::vector<std::string> arguments)
} }
} }
no_input_specified = !file_found;
// Based on what was read in open the sequence TODO // Based on what was read in open the sequence TODO
if (device != -1) if (device != -1)
{ {
@ -167,7 +169,10 @@ bool SequenceCapture::Open(std::vector<std::string> arguments)
{ {
return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy); return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy);
} }
// If none opened, return false
// If no input found return false and set a flag for it
no_input_specified = true;
return false; return false;
} }
@ -175,6 +180,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height,
{ {
INFO_STREAM("Attempting to read from webcam: " << device); INFO_STREAM("Attempting to read from webcam: " << device);
no_input_specified = false;
if (device < 0) if (device < 0)
{ {
std::cout << "Specify a valid device" << std::endl; std::cout << "Specify a valid device" << std::endl;
@ -232,6 +239,8 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy,
{ {
INFO_STREAM("Attempting to read from file: " << video_file); INFO_STREAM("Attempting to read from file: " << video_file);
no_input_specified = false;
latest_frame = cv::Mat(); latest_frame = cv::Mat();
latest_gray_frame = cv::Mat(); latest_gray_frame = cv::Mat();
@ -272,6 +281,8 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f
{ {
INFO_STREAM("Attempting to read from directory: " << directory); INFO_STREAM("Attempting to read from directory: " << directory);
no_input_specified = false;
image_files.clear(); image_files.clear();
boost::filesystem::path image_directory(directory); boost::filesystem::path image_directory(directory);