More changes to bring up to speed for GUI:
- Outputting gaze angle - Outputting eye landmarks
This commit is contained in:
Tadas Baltrusaitis
parent
e71d13a22a
commit
1f356eb8ae
6 changed files with 153 additions and 42 deletions
|
|
@ -204,13 +204,13 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
|
|||
|
||||
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
int num_landmarks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg);
|
||||
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg);
|
||||
|
||||
// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
|
||||
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success,
|
||||
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
|
||||
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
|
||||
const FaceAnalysis::FaceAnalyser& face_analyser);
|
||||
|
||||
int main (int argc, char **argv)
|
||||
|
|
@ -466,7 +466,9 @@ int main (int argc, char **argv)
|
|||
if (!output_files.empty())
|
||||
{
|
||||
output_file.open(output_files[f_n], ios_base::out);
|
||||
prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
|
||||
int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size();
|
||||
|
||||
prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
|
||||
}
|
||||
|
||||
// Saving the HOG features
|
||||
|
|
@ -548,11 +550,13 @@ int main (int argc, char **argv)
|
|||
// Gaze tracking, absolute gaze direction
|
||||
cv::Point3f gazeDirection0(0, 0, -1);
|
||||
cv::Point3f gazeDirection1(0, 0, -1);
|
||||
cv::Vec2d gazeAngle(0, 0);
|
||||
|
||||
if (det_parameters.track_gaze && detection_success && face_model.eye_model)
|
||||
{
|
||||
FaceAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true);
|
||||
FaceAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false);
|
||||
gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1);
|
||||
}
|
||||
|
||||
// Do face alignment
|
||||
|
|
@ -635,7 +639,7 @@ int main (int argc, char **argv)
|
|||
|
||||
// Output the landmarks, pose, gaze, parameters and AUs
|
||||
outputAllFeatures(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze,
|
||||
face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1,
|
||||
face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, gazeAngle,
|
||||
pose_estimate, fx, fy, cx, cy, face_analyser);
|
||||
|
||||
// output the tracked video
|
||||
|
|
@ -724,14 +728,23 @@ int main (int argc, char **argv)
|
|||
|
||||
void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
int num_landmarks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg)
|
||||
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg)
|
||||
{
|
||||
|
||||
*output_file << "frame, timestamp, confidence, success";
|
||||
|
||||
if (output_gaze)
|
||||
{
|
||||
*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z";
|
||||
*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
|
||||
|
||||
for (int i = 0; i < num_eye_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", eye_lmk_x_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_eye_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", eye_lmk_y_" << i;
|
||||
}
|
||||
}
|
||||
|
||||
if (output_pose)
|
||||
|
|
@ -741,11 +754,11 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
|
|||
|
||||
if (output_2D_landmarks)
|
||||
{
|
||||
for (int i = 0; i < num_landmarks; ++i)
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", x_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_landmarks; ++i)
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", y_" << i;
|
||||
}
|
||||
|
|
@ -753,15 +766,15 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
|
|||
|
||||
if (output_3D_landmarks)
|
||||
{
|
||||
for (int i = 0; i < num_landmarks; ++i)
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", X_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_landmarks; ++i)
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", Y_" << i;
|
||||
}
|
||||
for (int i = 0; i < num_landmarks; ++i)
|
||||
for (int i = 0; i < num_face_landmarks; ++i)
|
||||
{
|
||||
*output_file << ", Z_" << i;
|
||||
}
|
||||
|
|
@ -800,7 +813,7 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
|
|||
void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
|
||||
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
|
||||
const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success,
|
||||
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
|
||||
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
|
||||
const FaceAnalysis::FaceAnalyser& face_analyser)
|
||||
{
|
||||
|
||||
|
|
@ -813,6 +826,21 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
|
|||
{
|
||||
*output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z
|
||||
<< ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z;
|
||||
|
||||
// Output gaze angle (same format as head pose angle)
|
||||
*output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1];
|
||||
|
||||
// Output eye landmarks
|
||||
std::vector<cv::Point2d> eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(face_model);
|
||||
|
||||
for (size_t i = 0; i < eye_landmark_points.size(); ++i)
|
||||
{
|
||||
*output_file << ", " << eye_landmark_points[i].x;
|
||||
}
|
||||
for (size_t i = 0; i < eye_landmark_points.size(); ++i)
|
||||
{
|
||||
*output_file << ", " << eye_landmark_points[i].y;
|
||||
}
|
||||
}
|
||||
|
||||
// Output the estimated head pose
|
||||
|
|
|
|||
|
|
@ -44,5 +44,9 @@ namespace FaceAnalysis
|
|||
void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye);
|
||||
void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy);
|
||||
|
||||
// Getting the gaze angle in radians with respect to the world coordinates (camera plane), when looking ahead straight at camera plane the gaze angle will be (0,0)
|
||||
cv::Vec2d GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2);
|
||||
|
||||
|
||||
}
|
||||
#endif
|
||||
|
|
@ -133,6 +133,17 @@ void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po
|
|||
gaze_absolute = gazeVecAxis / norm(gazeVecAxis);
|
||||
}
|
||||
|
||||
cv::Vec2d FaceAnalysis::GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2)
|
||||
{
|
||||
|
||||
cv::Point3f gaze_vector = (gaze_vector_1 + gaze_vector_2) / 2;
|
||||
|
||||
double x_angle = atan2(gaze_vector.x, -gaze_vector.z);
|
||||
double y_angle = atan2(gaze_vector.y, -gaze_vector.z);
|
||||
|
||||
return cv::Vec2d(x_angle, y_angle);
|
||||
|
||||
}
|
||||
|
||||
void FaceAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -89,8 +89,13 @@ namespace LandmarkDetector
|
|||
vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy);
|
||||
void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar color, int thickness);
|
||||
|
||||
vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_<int>& visibilities);
|
||||
vector<cv::Point2d> CalculateLandmarks(CLNF& clnf_model);
|
||||
vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);
|
||||
vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model);
|
||||
vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model);
|
||||
|
||||
vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D);
|
||||
vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model);
|
||||
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model);
|
||||
void DrawLandmarks(cv::Mat img, vector<cv::Point> landmarks);
|
||||
|
||||
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);
|
||||
|
|
|
|||
|
|
@ -970,7 +970,7 @@ void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar
|
|||
}
|
||||
|
||||
// Computing landmarks (to be drawn later possibly)
|
||||
vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_<int>& visibilities)
|
||||
vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
|
||||
{
|
||||
int n = shape2D.rows / 2;
|
||||
vector<cv::Point2d> landmarks;
|
||||
|
|
@ -989,7 +989,7 @@ vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_
|
|||
}
|
||||
|
||||
// Computing landmarks (to be drawn later possibly)
|
||||
vector<cv::Point2d> CalculateLandmarks(cv::Mat img, const cv::Mat_<double>& shape2D)
|
||||
vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D)
|
||||
{
|
||||
|
||||
int n;
|
||||
|
|
@ -1023,15 +1023,73 @@ vector<cv::Point2d> CalculateLandmarks(cv::Mat img, const cv::Mat_<double>& shap
|
|||
}
|
||||
|
||||
// Computing landmarks (to be drawn later possibly)
|
||||
vector<cv::Point2d> CalculateLandmarks(CLNF& clnf_model)
|
||||
vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model)
|
||||
{
|
||||
return CalculateAllLandmarks(clnf_model.detected_landmarks);
|
||||
}
|
||||
|
||||
// Computing landmarks (to be drawn later possibly)
|
||||
vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model)
|
||||
{
|
||||
// If the detection was not successful no landmarks are visible
|
||||
if (clnf_model.detection_success)
|
||||
{
|
||||
int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
|
||||
// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
|
||||
return CalculateVisibleLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
|
||||
}
|
||||
else
|
||||
{
|
||||
return vector<cv::Point2d>();
|
||||
}
|
||||
}
|
||||
|
||||
// Computing eye landmarks (to be drawn later or in different interfaces)
|
||||
vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model)
|
||||
{
|
||||
|
||||
int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
|
||||
vector<cv::Point2d> to_return;
|
||||
// If the model has hierarchical updates draw those too
|
||||
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
|
||||
{
|
||||
|
||||
// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
|
||||
return CalculateLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
|
||||
if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 ||
|
||||
clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
|
||||
{
|
||||
|
||||
auto lmks = CalculateVisibleLandmarks(clnf_model.hierarchical_models[i]);
|
||||
for (auto lmk : lmks)
|
||||
{
|
||||
to_return.push_back(lmk);
|
||||
}
|
||||
}
|
||||
}
|
||||
return to_return;
|
||||
}
|
||||
|
||||
// Computing eye landmarks (to be drawn later or in different interfaces)
|
||||
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model)
|
||||
{
|
||||
|
||||
vector<cv::Point2d> to_return;
|
||||
// If the model has hierarchical updates draw those too
|
||||
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
|
||||
{
|
||||
|
||||
if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 ||
|
||||
clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
|
||||
{
|
||||
|
||||
auto lmks = CalculateAllLandmarks(clnf_model.hierarchical_models[i]);
|
||||
for (auto lmk : lmks)
|
||||
{
|
||||
to_return.push_back(lmk);
|
||||
}
|
||||
}
|
||||
}
|
||||
return to_return;
|
||||
}
|
||||
|
||||
|
||||
// Drawing landmarks on a face image
|
||||
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
|
||||
|
|
|
|||
|
|
@ -82,10 +82,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_'))
|
|||
xs = all_params(valid_frames, landmark_inds_x);
|
||||
ys = all_params(valid_frames, landmark_inds_y);
|
||||
|
||||
eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_'));
|
||||
eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_'));
|
||||
|
||||
eye_xs = all_params(valid_frames, eye_landmark_inds_x);
|
||||
eye_ys = all_params(valid_frames, eye_landmark_inds_y);
|
||||
|
||||
figure
|
||||
|
||||
for j = 1:size(xs,1)
|
||||
plot(xs(j,:), -ys(j,:), '.');
|
||||
hold on;
|
||||
plot(eye_xs(j,:), -eye_ys(j,:), '.r');
|
||||
hold off;
|
||||
|
||||
xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]);
|
||||
ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]);
|
||||
xlabel('x (px)');
|
||||
|
|
@ -141,21 +151,16 @@ title('Pose (rotation and translation)');
|
|||
xlabel('Time (s)');
|
||||
|
||||
%% Demo gaze
|
||||
gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_'));
|
||||
gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle'));
|
||||
|
||||
% Read gaze (x,y,z) for one eye and (x,y,z) for another
|
||||
gaze = all_params(valid_frames, gaze_inds);
|
||||
|
||||
% only picking left, right and up down views for visualisation
|
||||
gaze = (gaze(:,[1,2]) + gaze(:,[4,5]))/2;
|
||||
gaze(:,1) = smooth(gaze(:,1));
|
||||
gaze(:,2) = smooth(gaze(:,2));
|
||||
|
||||
plot(time, gaze(:,1), 'DisplayName', 'Left - right');
|
||||
hold on;
|
||||
plot(time, gaze(:,2), 'DisplayName', 'Up - down');
|
||||
xlabel('Time(s)') % x-axis label
|
||||
ylabel('Gaze vector size') % y-axis label
|
||||
ylabel('Angle radians') % y-axis label
|
||||
legend('show');
|
||||
hold off;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue