r
/
sustaining_gazes
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More changes to bring up to speed for GUI: 

- Outputting gaze angle
- Outputting eye landmarks
This commit is contained in:
Tadas Baltrusaitis 2017-10-21 20:40:18 +01:00
parent e71d13a22a
commit 1f356eb8ae
6 changed files with 153 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
exe/FeatureExtraction/FeatureExtraction.cpp
View File

@ -204,13 +204,13 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_landmarks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg);
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg);
// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success,
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
const FaceAnalysis::FaceAnalyser& face_analyser);
int main (int argc, char **argv)
@ -466,7 +466,9 @@ int main (int argc, char **argv)
if (!output_files.empty())
{
output_file.open(output_files[f_n], ios_base::out);
prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size();
prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
}
// Saving the HOG features
@ -548,11 +550,13 @@ int main (int argc, char **argv)
// Gaze tracking, absolute gaze direction
cv::Point3f gazeDirection0(0, 0, -1);
cv::Point3f gazeDirection1(0, 0, -1);
cv::Vec2d gazeAngle(0, 0);
if (det_parameters.track_gaze && detection_success && face_model.eye_model)
{
FaceAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true);
FaceAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false);
gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1);
}
// Do face alignment
@ -635,7 +639,7 @@ int main (int argc, char **argv)
// Output the landmarks, pose, gaze, parameters and AUs
outputAllFeatures(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze,
face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1,
face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, gazeAngle,
pose_estimate, fx, fy, cx, cy, face_analyser);
// output the tracked video
@ -724,14 +728,23 @@ int main (int argc, char **argv)
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_landmarks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg)
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg)
{
*output_file << "frame, timestamp, confidence, success";
if (output_gaze)
{
*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z";
*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
for (int i = 0; i < num_eye_landmarks; ++i)
{
*output_file << ", eye_lmk_x_" << i;
}
for (int i = 0; i < num_eye_landmarks; ++i)
{
*output_file << ", eye_lmk_y_" << i;
}
}
if (output_pose)
@ -741,11 +754,11 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
if (output_2D_landmarks)
{
for (int i = 0; i < num_landmarks; ++i)
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", x_" << i;
}
for (int i = 0; i < num_landmarks; ++i)
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", y_" << i;
}
@ -753,15 +766,15 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
if (output_3D_landmarks)
{
for (int i = 0; i < num_landmarks; ++i)
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", X_" << i;
}
for (int i = 0; i < num_landmarks; ++i)
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", Y_" << i;
}
for (int i = 0; i < num_landmarks; ++i)
for (int i = 0; i < num_face_landmarks; ++i)
{
*output_file << ", Z_" << i;
}
@ -800,7 +813,7 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success,
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
const FaceAnalysis::FaceAnalyser& face_analyser)
{
@ -813,6 +826,21 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
{
*output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z
<< ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z;
// Output gaze angle (same format as head pose angle)
*output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1];
// Output eye landmarks
std::vector<cv::Point2d> eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(face_model);
for (size_t i = 0; i < eye_landmark_points.size(); ++i)
{
*output_file << ", " << eye_landmark_points[i].x;
}
for (size_t i = 0; i < eye_landmark_points.size(); ++i)
{
*output_file << ", " << eye_landmark_points[i].y;
}
}
// Output the estimated head pose

4
lib/local/FaceAnalyser/include/GazeEstimation.h
View File

@ -44,5 +44,9 @@ namespace FaceAnalysis
void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye);
void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy);
// Getting the gaze angle in radians with respect to the world coordinates (camera plane), when looking ahead straight at camera plane the gaze angle will be (0,0)
cv::Vec2d GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2);
}
#endif

11
lib/local/FaceAnalyser/src/GazeEstimation.cpp
View File

@ -133,6 +133,17 @@ void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po
gaze_absolute = gazeVecAxis / norm(gazeVecAxis);
}
cv::Vec2d FaceAnalysis::GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2)
{
cv::Point3f gaze_vector = (gaze_vector_1 + gaze_vector_2) / 2;
double x_angle = atan2(gaze_vector.x, -gaze_vector.z);
double y_angle = atan2(gaze_vector.y, -gaze_vector.z);
return cv::Vec2d(x_angle, y_angle);
}
void FaceAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy)
{

9
lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h
View File

@ -89,8 +89,13 @@ namespace LandmarkDetector
vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy);
void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar color, int thickness);
vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_<int>& visibilities);
vector<cv::Point2d> CalculateLandmarks(CLNF& clnf_model);
vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);
vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model);
vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model);
vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D);
vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model);
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model);
void DrawLandmarks(cv::Mat img, vector<cv::Point> landmarks);
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);

100
lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp
View File

@ -970,16 +970,16 @@ void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar
}
// Computing landmarks (to be drawn later possibly)
vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_<int>& visibilities)
vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
{
int n = shape2D.rows/2;
int n = shape2D.rows / 2;
vector<cv::Point2d> landmarks;
for( int i = 0; i < n; ++i)
{
if(visibilities.at<int>(i))
for (int i = 0; i < n; ++i)
{
if (visibilities.at<int>(i))
{
cv::Point2d featurePoint(shape2D.at<double>(i), shape2D.at<double>(i +n));
cv::Point2d featurePoint(shape2D.at<double>(i), shape2D.at<double>(i + n));
landmarks.push_back(featurePoint);
}
@ -989,27 +989,27 @@ vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_
}
// Computing landmarks (to be drawn later possibly)
vector<cv::Point2d> CalculateLandmarks(cv::Mat img, const cv::Mat_<double>& shape2D)
vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D)
{
int n;
vector<cv::Point2d> landmarks;
if(shape2D.cols == 2)
if (shape2D.cols == 2)
{
n = shape2D.rows;
}
else if(shape2D.cols == 1)
else if (shape2D.cols == 1)
{
n = shape2D.rows/2;
n = shape2D.rows / 2;
}
for( int i = 0; i < n; ++i)
{
for (int i = 0; i < n; ++i)
{
cv::Point2d featurePoint;
if(shape2D.cols == 1)
if (shape2D.cols == 1)
{
featurePoint = cv::Point2d(shape2D.at<double>(i), shape2D.at<double>(i +n));
featurePoint = cv::Point2d(shape2D.at<double>(i), shape2D.at<double>(i + n));
}
else
{
@ -1018,21 +1018,79 @@ vector<cv::Point2d> CalculateLandmarks(cv::Mat img, const cv::Mat_<double>& shap
landmarks.push_back(featurePoint);
}
return landmarks;
}
// Computing landmarks (to be drawn later possibly)
vector<cv::Point2d> CalculateLandmarks(CLNF& clnf_model)
vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model)
{
return CalculateAllLandmarks(clnf_model.detected_landmarks);
}
// Computing landmarks (to be drawn later possibly)
vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model)
{
// If the detection was not successful no landmarks are visible
if (clnf_model.detection_success)
{
int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
return CalculateVisibleLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
}
else
{
return vector<cv::Point2d>();
}
}
// Computing eye landmarks (to be drawn later or in different interfaces)
vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model)
{
int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
vector<cv::Point2d> to_return;
// If the model has hierarchical updates draw those too
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
{
// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
return CalculateLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 ||
clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
{
auto lmks = CalculateVisibleLandmarks(clnf_model.hierarchical_models[i]);
for (auto lmk : lmks)
{
to_return.push_back(lmk);
}
}
}
return to_return;
}
// Computing eye landmarks (to be drawn later or in different interfaces)
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model)
{
vector<cv::Point2d> to_return;
// If the model has hierarchical updates draw those too
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
{
if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 ||
clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
{
auto lmks = CalculateAllLandmarks(clnf_model.hierarchical_models[i]);
for (auto lmk : lmks)
{
to_return.push_back(lmk);
}
}
}
return to_return;
}
// Drawing landmarks on a face image
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
{

19
matlab_runners/Demos/feature_extraction_demo_vid.m
View File

@ -82,10 +82,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_'))
xs = all_params(valid_frames, landmark_inds_x);
ys = all_params(valid_frames, landmark_inds_y);
eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_'));
eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_'));
eye_xs = all_params(valid_frames, eye_landmark_inds_x);
eye_ys = all_params(valid_frames, eye_landmark_inds_y);
figure
for j = 1:size(xs,1)
plot(xs(j,:), -ys(j,:), '.');
hold on;
plot(eye_xs(j,:), -eye_ys(j,:), '.r');
hold off;
xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]);
ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]);
xlabel('x (px)');
@ -141,21 +151,16 @@ title('Pose (rotation and translation)');
xlabel('Time (s)');
%% Demo gaze
gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_'));
gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle'));
% Read gaze (x,y,z) for one eye and (x,y,z) for another
gaze = all_params(valid_frames, gaze_inds);
% only picking left, right and up down views for visualisation
gaze = (gaze(:,[1,2]) + gaze(:,[4,5]))/2;
gaze(:,1) = smooth(gaze(:,1));
gaze(:,2) = smooth(gaze(:,2));
plot(time, gaze(:,1), 'DisplayName', 'Left - right');
hold on;
plot(time, gaze(:,2), 'DisplayName', 'Up - down');
xlabel('Time(s)') % x-axis label
ylabel('Gaze vector size') % y-axis label
ylabel('Angle radians') % y-axis label
legend('show');
hold off;