diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 4a5fe8b..c13a7a6 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -204,13 +204,13 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_landmarks, int num_model_modes, vector au_names_class, vector au_names_reg); + int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector au_names_class, vector au_names_reg); // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow) void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success, - cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, + cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, const FaceAnalysis::FaceAnalyser& face_analyser); int main (int argc, char **argv) @@ -466,7 +466,9 @@ int main (int argc, char **argv) if (!output_files.empty()) { output_file.open(output_files[f_n], ios_base::out); - prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); + int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); + + prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); } // Saving the HOG features @@ -548,11 +550,13 @@ int main (int argc, char **argv) // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); + cv::Vec2d gazeAngle(0, 0); if (det_parameters.track_gaze && detection_success && face_model.eye_model) { FaceAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true); FaceAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false); + gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } // Do face alignment @@ -635,7 +639,7 @@ int main (int argc, char **argv) // Output the landmarks, pose, gaze, parameters and AUs outputAllFeatures(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, - face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, + face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, gazeAngle, pose_estimate, fx, fy, cx, cy, face_analyser); // output the tracked video @@ -724,14 +728,23 @@ int main (int argc, char **argv) void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_landmarks, int num_model_modes, vector au_names_class, vector au_names_reg) + int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector au_names_class, vector au_names_reg) { *output_file << "frame, timestamp, confidence, success"; if (output_gaze) { - *output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z"; + *output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y"; + + for (int i = 0; i < num_eye_landmarks; ++i) + { + *output_file << ", eye_lmk_x_" << i; + } + for (int i = 0; i < num_eye_landmarks; ++i) + { + *output_file << ", eye_lmk_y_" << i; + } } if (output_pose) @@ -741,11 +754,11 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo if (output_2D_landmarks) { - for (int i = 0; i < num_landmarks; ++i) + for (int i = 0; i < num_face_landmarks; ++i) { *output_file << ", x_" << i; } - for (int i = 0; i < num_landmarks; ++i) + for (int i = 0; i < num_face_landmarks; ++i) { *output_file << ", y_" << i; } @@ -753,15 +766,15 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo if (output_3D_landmarks) { - for (int i = 0; i < num_landmarks; ++i) + for (int i = 0; i < num_face_landmarks; ++i) { *output_file << ", X_" << i; } - for (int i = 0; i < num_landmarks; ++i) + for (int i = 0; i < num_face_landmarks; ++i) { *output_file << ", Y_" << i; } - for (int i = 0; i < num_landmarks; ++i) + for (int i = 0; i < num_face_landmarks; ++i) { *output_file << ", Z_" << i; } @@ -800,7 +813,7 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success, - cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, + cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, const FaceAnalysis::FaceAnalyser& face_analyser) { @@ -813,6 +826,21 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo { *output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z << ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z; + + // Output gaze angle (same format as head pose angle) + *output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1]; + + // Output eye landmarks + std::vector eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(face_model); + + for (size_t i = 0; i < eye_landmark_points.size(); ++i) + { + *output_file << ", " << eye_landmark_points[i].x; + } + for (size_t i = 0; i < eye_landmark_points.size(); ++i) + { + *output_file << ", " << eye_landmark_points[i].y; + } } // Output the estimated head pose diff --git a/lib/local/FaceAnalyser/include/GazeEstimation.h b/lib/local/FaceAnalyser/include/GazeEstimation.h index c0599a2..c8101e2 100644 --- a/lib/local/FaceAnalyser/include/GazeEstimation.h +++ b/lib/local/FaceAnalyser/include/GazeEstimation.h @@ -44,5 +44,9 @@ namespace FaceAnalysis void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye); void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy); + // Getting the gaze angle in radians with respect to the world coordinates (camera plane), when looking ahead straight at camera plane the gaze angle will be (0,0) + cv::Vec2d GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2); + + } #endif \ No newline at end of file diff --git a/lib/local/FaceAnalyser/src/GazeEstimation.cpp b/lib/local/FaceAnalyser/src/GazeEstimation.cpp index c7386d3..dbfd433 100644 --- a/lib/local/FaceAnalyser/src/GazeEstimation.cpp +++ b/lib/local/FaceAnalyser/src/GazeEstimation.cpp @@ -133,6 +133,17 @@ void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po gaze_absolute = gazeVecAxis / norm(gazeVecAxis); } +cv::Vec2d FaceAnalysis::GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2) +{ + + cv::Point3f gaze_vector = (gaze_vector_1 + gaze_vector_2) / 2; + + double x_angle = atan2(gaze_vector.x, -gaze_vector.z); + double y_angle = atan2(gaze_vector.y, -gaze_vector.z); + + return cv::Vec2d(x_angle, y_angle); + +} void FaceAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy) { diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 10d2c63..e102964 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -89,8 +89,13 @@ namespace LandmarkDetector vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness); - vector CalculateLandmarks(const cv::Mat_& shape2D, cv::Mat_& visibilities); - vector CalculateLandmarks(CLNF& clnf_model); + vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities); + vector CalculateVisibleLandmarks(const CLNF& clnf_model); + vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model); + + vector CalculateAllLandmarks(const cv::Mat_& shape2D); + vector CalculateAllLandmarks(const CLNF& clnf_model); + vector CalculateAllEyeLandmarks(const CLNF& clnf_model); void DrawLandmarks(cv::Mat img, vector landmarks); void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 6962e52..e9cb5eb 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -970,16 +970,16 @@ void DrawBox(vector> lines, cv::Mat image, cv::Scalar } // Computing landmarks (to be drawn later possibly) -vector CalculateLandmarks(const cv::Mat_& shape2D, cv::Mat_& visibilities) +vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities) { - int n = shape2D.rows/2; + int n = shape2D.rows / 2; vector landmarks; - for( int i = 0; i < n; ++i) - { - if(visibilities.at(i)) + for (int i = 0; i < n; ++i) + { + if (visibilities.at(i)) { - cv::Point2d featurePoint(shape2D.at(i), shape2D.at(i +n)); + cv::Point2d featurePoint(shape2D.at(i), shape2D.at(i + n)); landmarks.push_back(featurePoint); } @@ -989,27 +989,27 @@ vector CalculateLandmarks(const cv::Mat_& shape2D, cv::Mat_ } // Computing landmarks (to be drawn later possibly) -vector CalculateLandmarks(cv::Mat img, const cv::Mat_& shape2D) +vector CalculateAllLandmarks(const cv::Mat_& shape2D) { - + int n; vector landmarks; - - if(shape2D.cols == 2) + + if (shape2D.cols == 2) { n = shape2D.rows; } - else if(shape2D.cols == 1) + else if (shape2D.cols == 1) { - n = shape2D.rows/2; + n = shape2D.rows / 2; } - for( int i = 0; i < n; ++i) - { + for (int i = 0; i < n; ++i) + { cv::Point2d featurePoint; - if(shape2D.cols == 1) + if (shape2D.cols == 1) { - featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i +n)); + featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i + n)); } else { @@ -1018,21 +1018,79 @@ vector CalculateLandmarks(cv::Mat img, const cv::Mat_& shap landmarks.push_back(featurePoint); } - + return landmarks; } // Computing landmarks (to be drawn later possibly) -vector CalculateLandmarks(CLNF& clnf_model) +vector CalculateAllLandmarks(const CLNF& clnf_model) +{ + return CalculateAllLandmarks(clnf_model.detected_landmarks); +} + +// Computing landmarks (to be drawn later possibly) +vector CalculateVisibleLandmarks(const CLNF& clnf_model) +{ + // If the detection was not successful no landmarks are visible + if (clnf_model.detection_success) + { + int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); + // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation + return CalculateVisibleLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); + } + else + { + return vector(); + } +} + +// Computing eye landmarks (to be drawn later or in different interfaces) +vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model) { - int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); + vector to_return; + // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { - // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation - return CalculateLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 || + clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + auto lmks = CalculateVisibleLandmarks(clnf_model.hierarchical_models[i]); + for (auto lmk : lmks) + { + to_return.push_back(lmk); + } + } + } + return to_return; } +// Computing eye landmarks (to be drawn later or in different interfaces) +vector CalculateAllEyeLandmarks(const CLNF& clnf_model) +{ + + vector to_return; + // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 || + clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + + auto lmks = CalculateAllLandmarks(clnf_model.hierarchical_models[i]); + for (auto lmk : lmks) + { + to_return.push_back(lmk); + } + } + } + return to_return; +} + + // Drawing landmarks on a face image void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) { diff --git a/matlab_runners/Demos/feature_extraction_demo_vid.m b/matlab_runners/Demos/feature_extraction_demo_vid.m index bf2e2dd..32e5505 100644 --- a/matlab_runners/Demos/feature_extraction_demo_vid.m +++ b/matlab_runners/Demos/feature_extraction_demo_vid.m @@ -82,10 +82,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_')) xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); + figure for j = 1:size(xs,1) plot(xs(j,:), -ys(j,:), '.'); + hold on; + plot(eye_xs(j,:), -eye_ys(j,:), '.r'); + hold off; + xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]); ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]); xlabel('x (px)'); @@ -141,21 +151,16 @@ title('Pose (rotation and translation)'); xlabel('Time (s)'); %% Demo gaze -gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_')); +gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle')); % Read gaze (x,y,z) for one eye and (x,y,z) for another gaze = all_params(valid_frames, gaze_inds); -% only picking left, right and up down views for visualisation -gaze = (gaze(:,[1,2]) + gaze(:,[4,5]))/2; -gaze(:,1) = smooth(gaze(:,1)); -gaze(:,2) = smooth(gaze(:,2)); - plot(time, gaze(:,1), 'DisplayName', 'Left - right'); hold on; plot(time, gaze(:,2), 'DisplayName', 'Up - down'); xlabel('Time(s)') % x-axis label -ylabel('Gaze vector size') % y-axis label +ylabel('Angle radians') % y-axis label legend('show'); hold off;