From 729d9a9a85e08aeafd907ba21973fa3453eddee2 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 22 Oct 2017 20:02:54 +0100 Subject: [PATCH] Adding ability to output eye landmark locations during image processing. --- exe/FaceLandmarkImg/FaceLandmarkImg.cpp | 36 ++++++++++++++----- matlab_runners/Full_test_suite.m | 2 +- .../run_biwi_experiment.m | 2 +- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 08e2776..ced96ac 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -160,15 +160,16 @@ void write_out_pose_landmarks(const string& outfeatures, const cv::Mat_& featuresFile << "}" << endl; // Do the pose and eye gaze if present as well - featuresFile << "gaze: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; + featuresFile << "gaze_vec: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; featuresFile << "{" << endl; featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl; featuresFile << "}" << endl; + featuresFile.close(); } } -void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1, std::vector> au_intensities, std::vector> au_occurences) +void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1, const cv::Vec2d gaze_angle, std::vector> au_intensities, std::vector> au_occurences) { create_directory_from_file(outfeatures); std::ofstream featuresFile; @@ -177,7 +178,7 @@ void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF if (featuresFile.is_open()) { int n = clnf_model.patch_experts.visibilities[0][0].rows; - featuresFile << "version: 1" << endl; + featuresFile << "version: 2" << endl; featuresFile << "npoints: " << n << endl; featuresFile << "{" << endl; @@ -194,12 +195,28 @@ void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl; featuresFile << "}" << endl; - // Do the pose and eye gaze if present as well featuresFile << "gaze: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; featuresFile << "{" << endl; featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl; featuresFile << "}" << endl; + featuresFile << "gaze: angle_x, angle_y: " << endl; + featuresFile << "{" << endl; + featuresFile << gaze_angle[0] << " " << gaze_angle[1] << endl; + featuresFile << "}" << endl; + + std::vector eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(clnf_model); + + featuresFile << "eye_lmks: " << eye_landmark_points.size() << endl; + featuresFile << "{" << endl; + + for (int i = 0; i < eye_landmark_points.size(); ++i) + { + // Use matlab format, so + 1 + featuresFile << eye_landmark_points[i].x + 1 << " " << eye_landmark_points[i].y + 1 << endl; + } + featuresFile << "}" << endl; + // Do the au intensities featuresFile << "au intensities: " << au_intensities.size() << endl; featuresFile << "{" << endl; @@ -451,12 +468,13 @@ int main (int argc, char **argv) // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); + cv::Vec2d gazeAngle(0, 0); if (success && det_parameters.track_gaze) { FaceAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); - + gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false); @@ -477,7 +495,7 @@ int main (int argc, char **argv) boost::filesystem::path fname = out_feat_path.filename().replace_extension(""); boost::filesystem::path ext = out_feat_path.extension(); string outfeatures = dir.string() + preferredSlash + fname.string() + string(name) + ext.string(); - write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second); + write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, gazeAngle, ActionUnits.first, ActionUnits.second); } if (!output_pose_locations.empty()) @@ -567,11 +585,13 @@ int main (int argc, char **argv) // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); - + cv::Vec2d gazeAngle(0, 0); + if (det_parameters.track_gaze) { FaceAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); + gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false); @@ -580,7 +600,7 @@ int main (int argc, char **argv) if(!output_landmark_locations.empty()) { string outfeatures = output_landmark_locations.at(i); - write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second); + write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, gazeAngle, ActionUnits.first, ActionUnits.second); } // Writing out the detected landmarks diff --git a/matlab_runners/Full_test_suite.m b/matlab_runners/Full_test_suite.m index 0b3e8a4..c6f5d0f 100644 --- a/matlab_runners/Full_test_suite.m +++ b/matlab_runners/Full_test_suite.m @@ -5,7 +5,7 @@ tic %% Head pose cd('Head Pose Experiments'); run_head_pose_tests_OpenFace; -assert(median(all_errors_biwi_OF(:)) < 2.7); +assert(median(all_errors_biwi_OF(:)) < 2.8); assert(median(all_errors_bu_OF(:)) < 2.2); assert(median(all_errors_ict_OF(:)) < 2.1); cd('../'); diff --git a/matlab_runners/Head Pose Experiments/run_biwi_experiment.m b/matlab_runners/Head Pose Experiments/run_biwi_experiment.m index cdf7f0f..8a564cf 100644 --- a/matlab_runners/Head Pose Experiments/run_biwi_experiment.m +++ b/matlab_runners/Head Pose Experiments/run_biwi_experiment.m @@ -44,7 +44,7 @@ for i=3 + offset:numTogether:numel(dbSeqDir) command = cat(2, command, [' -ov "' outputVideo '"']); end end - command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track -camera_coord ']); + command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track ']); if(any(strcmp('model', varargin))) command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']);