r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Simplifying the visualizer and using it in FaceLandmarkVid

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-11-13 17:30:19 +00:00
parent fa48372f4c
commit 9f4bf5724f
4 changed files with 33 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
exe/FaceLandmarkVid/FaceLandmarkVid.cpp
View file

@ -47,6 +47,8 @@
#include <opencv2/highgui/highgui.hpp> #include <opencv2/highgui/highgui.hpp>
#include <SequenceCapture.h> #include <SequenceCapture.h>
#include <Visualizer.h>
#include <VisualizationUtils.h>
// Boost includes // Boost includes
#include <filesystem.hpp> #include <filesystem.hpp>
@ -84,65 +86,6 @@ vector<string> get_arguments(int argc, char **argv)
return arguments; return arguments;
} }
// Some globals for tracking timing information for visualisation
double fps_tracker = -1.0;
int64 t0 = 0;
int frame_count = 0;
// Visualising the results, TODO move to separate
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy)
{
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
double detection_certainty = face_model.detection_certainty;
bool detection_success = face_model.detection_success;
double visualisation_boundary = 0.4;
// Only draw if the reliability is reasonable, the value is slightly ad-hoc
if (detection_certainty > visualisation_boundary)
{
LandmarkDetector::Draw(captured_image, face_model);
double vis_certainty = detection_certainty;
if (vis_certainty > 1)
vis_certainty = 1;
// Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking
vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary);
// A rough heuristic for box around the face width
int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0);
cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy);
// Draw it in reddish if uncertain, blueish if certain
LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar(vis_certainty*255.0, 0, (1 - vis_certainty) * 255), thickness, fx, fy, cx, cy);
if (det_parameters.track_gaze && detection_success && face_model.eye_model)
{
GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy);
}
}
// Work out the framerate TODO
if (frame_count % 10 == 0)
{
double t1 = cv::getTickCount();
fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency());
t0 = t1;
}
// Write out the framerate on the image before displaying it
char fpsC[255];
std::sprintf(fpsC, "%d", (int)fps_tracker);
string fpsSt("FPS:");
fpsSt += fpsC;
cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
frame_count++;
}
int main (int argc, char **argv) int main (int argc, char **argv)
{ {
@ -152,11 +95,18 @@ int main (int argc, char **argv)
det_parameters.track_gaze = true; det_parameters.track_gaze = true;
// The modules that are being used for tracking // The modules that are being used for tracking
LandmarkDetector::CLNF clnf_model(det_parameters.model_location); LandmarkDetector::CLNF face_model(det_parameters.model_location);
// Open a sequence // Open a sequence
Utilities::SequenceCapture sequence_reader; Utilities::SequenceCapture sequence_reader;
// A utility for visualizing the results
Utilities::Visualizer visualizer(arguments);
// Tracking FPS for visualization
Utilities::FpsTracker fps_tracker;
fps_tracker.AddFrame();
while (true) // this is not a for loop as we might also be reading from a webcam while (true) // this is not a for loop as we might also be reading from a webcam
{ {
@ -186,23 +136,35 @@ int main (int argc, char **argv)
cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame(); cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
// The actual facial landmark detection / tracking // The actual facial landmark detection / tracking
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_model, det_parameters); bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters);
// Visualising the results // Visualising the results
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
double detection_certainty = clnf_model.detection_certainty; double detection_certainty = face_model.detection_certainty;
// Gaze tracking, absolute gaze direction // Gaze tracking, absolute gaze direction
cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection0(0, 0, -1);
cv::Point3f gazeDirection1(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1);
if (det_parameters.track_gaze && detection_success && clnf_model.eye_model) if (det_parameters.track_gaze && detection_success && face_model.eye_model)
{ {
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
} }
visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); // Work out the pose of the head from the tracked model
cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// Keeping track of FPS
fps_tracker.AddFrame();
// Displaying the tracking visualizations
visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success);
visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
visualizer.SetFps(fps_tracker.GetFPS());
visualizer.ShowObservation();
// detect key presses // detect key presses
char character_press = cv::waitKey(1); char character_press = cv::waitKey(1);
@ -210,7 +172,7 @@ int main (int argc, char **argv)
// restart the tracker // restart the tracker
if (character_press == 'r') if (character_press == 'r')
{ {
clnf_model.Reset(); face_model.Reset();
} }
// quit the application // quit the application
else if (character_press == 'q') else if (character_press == 'q')
@ -221,7 +183,7 @@ int main (int argc, char **argv)
} }
// Reset the model, for the next video // Reset the model, for the next video
clnf_model.Reset(); face_model.Reset();
} }
return 0; return 0;

2
exe/FeatureExtraction/FeatureExtraction.cpp
View file

@ -186,7 +186,7 @@ int main (int argc, char **argv)
visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success); visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success);
visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
visualizer.SetFps(fps_tracker.GetFPS()); visualizer.SetFps(fps_tracker.GetFPS());
visualizer.ShowObservation(); visualizer.ShowObservation();

2
lib/local/Utilities/include/Visualizer.h
View file

@ -72,7 +72,7 @@ namespace Utilities
void SetObservationPose(const cv::Vec6d& pose, double confidence); void SetObservationPose(const cv::Vec6d& pose, double confidence);
// Gaze related observations // Gaze related observations
void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence); void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2d>& eye_landmarks, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence);
// Face alignment related observations // Face alignment related observations
void SetObservationFaceAlign(const cv::Mat& aligned_face); void SetObservationFaceAlign(const cv::Mat& aligned_face);

2
lib/local/Utilities/src/Visualizer.cpp
View file

@ -157,7 +157,7 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
} }
// Eye gaze infomration drawing, first of eye landmarks then of gaze // Eye gaze infomration drawing, first of eye landmarks then of gaze
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence) void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence)
{ {
if(confidence > visualisation_boundary) if(confidence > visualisation_boundary)
{ {