From 5ce0a8d4bfa0524d61c2d4d80e67649ee1e7b2bf Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 12 Nov 2017 10:55:47 +0000 Subject: [PATCH] More visualization cleanup and removal of redundant code. --- exe/FeatureExtraction/FeatureExtraction.cpp | 26 +--- lib/local/GazeAnalyser/GazeAnalyser.vcxproj | 8 +- lib/local/GazeAnalyser/src/GazeEstimation.cpp | 6 +- .../include/LandmarkDetectorUtils.h | 8 +- .../src/LandmarkDetectorUtils.cpp | 146 ------------------ lib/local/Utilities/include/RotationHelpers.h | 42 +++++ .../Utilities/include/VisualizationUtils.h | 2 - lib/local/Utilities/include/Visualizer.h | 2 +- lib/local/Utilities/src/RecorderOpenFace.cpp | 4 +- .../Utilities/src/VisualizationUtils.cpp | 40 ----- lib/local/Utilities/src/Visualizer.cpp | 57 +++---- 11 files changed, 80 insertions(+), 261 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 2d832bd..96bb3b1 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -206,37 +206,17 @@ int main (int argc, char **argv) // Displaying the tracking visualizations visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); visualizer.SetObservationFaceAlign(sim_warped_img); - visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model)); + visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success); - visualizer.SetObservationPose(pose_estimate); + visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.ShowObservation(); - if(!det_parameters.quiet_mode) - { - if (recording_params.outputTrackedVideo() || visualize_track) - { - cv::namedWindow("tracking_result", 1); - cv::imshow("tracking_result", captured_image); - cv::waitKey(1); - } - if (visualize_align) - { - cv::imshow("sim_warp", sim_warped_img); - } - if (visualize_hog) - { - cv::Mat_ hog_descriptor_vis; - FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_vis); - cv::imshow("hog", hog_descriptor_vis); - } - } - // Setting up the recorder output open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); - open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model)); + open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); open_face_rec.SetObservationPose(pose_estimate); diff --git a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj index 059d064..4e1a3ce 100644 --- a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj +++ b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj @@ -91,7 +91,7 @@ Level3 Disabled true - ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 @@ -100,7 +100,7 @@ Level3 Disabled true - ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) AdvancedVectorExtensions @@ -113,7 +113,7 @@ true - ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) true WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) StreamingSIMDExtensions2 @@ -131,7 +131,7 @@ true - ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) true WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) AdvancedVectorExtensions diff --git a/lib/local/GazeAnalyser/src/GazeEstimation.cpp b/lib/local/GazeAnalyser/src/GazeEstimation.cpp index 85346fd..04a3753 100644 --- a/lib/local/GazeAnalyser/src/GazeEstimation.cpp +++ b/lib/local/GazeAnalyser/src/GazeEstimation.cpp @@ -42,7 +42,7 @@ #include "LandmarkDetectorUtils.h" #include "LandmarkDetectorFunc.h" -#include "Utilities.h" +#include "RotationHelpers.h" using namespace std; @@ -185,12 +185,12 @@ void GazeAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_mode cv::Mat_ proj_points; cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); - LandmarkDetector::Project(proj_points, mesh_0, fx, fy, cx, cy); + Utilities::Project(proj_points, mesh_0, fx, fy, cx, cy); cv::line(img, cv::Point(cvRound(proj_points.at(0,0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(0, 1) * (double)gaze_draw_multiplier)), cv::Point(cvRound(proj_points.at(1, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(1, 1) * (double)gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits); cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); - LandmarkDetector::Project(proj_points, mesh_1, fx, fy, cx, cy); + Utilities::Project(proj_points, mesh_1, fx, fy, cx, cy); cv::line(img, cv::Point(cvRound(proj_points.at(0, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(0, 1) * (double)gaze_draw_multiplier)), cv::Point(cvRound(proj_points.at(1, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(1, 1) * (double)gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits); } \ No newline at end of file diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 1157259..a28bdcd 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -54,8 +54,6 @@ namespace LandmarkDetector //============================================================================================= void get_video_input_output_params(vector &input_video_file, vector &output_files, string &output_codec, vector &arguments); - void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments); - void get_image_input_output_params(vector &input_image_files, vector &output_feature_files, vector &output_pose_files, vector &output_image_files, vector> &input_bounding_boxes, vector &arguments); @@ -88,11 +86,7 @@ namespace LandmarkDetector vector CalculateAllLandmarks(const cv::Mat_& shape2D); vector CalculateAllLandmarks(const CLNF& clnf_model); vector CalculateAllEyeLandmarks(const CLNF& clnf_model); - void DrawLandmarks(cv::Mat img, vector landmarks); - - void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); - void Draw(cv::Mat img, const cv::Mat_& shape2D); - void Draw(cv::Mat img, const CLNF& clnf_model); + vector Calculate3DEyeLandmarks(const CLNF& clnf_model, double fx, double fy, double cx, double cy); //============================================================================ // Face detection helpers diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 458a6ec..54a2558 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -847,152 +847,6 @@ vector CalculateAllEyeLandmarks(const CLNF& clnf_model) return to_return; } - -// Drawing landmarks on a face image -void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) -{ - int n = shape2D.rows/2; - - - // Drawing feature points - if(n >= 66) - { - for( int i = 0; i < n; ++i) - { - if(visibilities.at(i)) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.0* ((double)img.cols) / 640.0); - - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - - } - } - } - else if(n == 28) // drawing eyes - { - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; - - int next_point = i + 1; - if(i == 7) - next_point = 0; - if(i == 19) - next_point = 8; - if(i == 27) - next_point = 20; - - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); - if( i < 8 || i > 19) - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - else - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); - - - } - } - else if(n == 6) - { - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; - - int next_point = i + 1; - if(i == 5) - next_point = 0; - - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - } - } -} - -// Drawing landmarks on a face image -void Draw(cv::Mat img, const cv::Mat_& shape2D) -{ - - int n; - - if(shape2D.cols == 2) - { - n = shape2D.rows; - } - else if(shape2D.cols == 1) - { - n = shape2D.rows/2; - } - - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint; - if(shape2D.cols == 1) - { - featurePoint = cv::Point(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - } - else - { - featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * (double)draw_multiplier), cvRound(shape2D.at(i, 1) * (double)draw_multiplier)); - } - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - - } - -} - -// Drawing detected landmarks on a face image -void Draw(cv::Mat img, const CLNF& clnf_model) -{ - - int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); - - // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation - Draw(img, clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); - - // If the model has hierarchical updates draw those too - for(size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) - { - if(clnf_model.hierarchical_models[i].pdm.NumberOfPoints() != clnf_model.hierarchical_mapping[i].size()) - { - Draw(img, clnf_model.hierarchical_models[i]); - } - } -} - -void DrawLandmarks(cv::Mat img, vector landmarks) -{ - for(cv::Point p : landmarks) - { - - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - - cv::circle(img, p, 1, cv::Scalar(0,0,255), thickness, CV_AA); - cv::circle(img, p, 1, cv::Scalar(255,0,0), thickness_2, CV_AA); - } - -} - - - //=========================================================================== //============================================================================ diff --git a/lib/local/Utilities/include/RotationHelpers.h b/lib/local/Utilities/include/RotationHelpers.h index c416185..242a0a1 100644 --- a/lib/local/Utilities/include/RotationHelpers.h +++ b/lib/local/Utilities/include/RotationHelpers.h @@ -114,5 +114,47 @@ namespace Utilities cv::Rodrigues(rotation_matrix, axis_angle); return axis_angle; } + + // Generally useful 3D functions + static void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) + { + dest = cv::Mat_(mesh.rows, 2, 0.0); + + int num_points = mesh.rows; + + double X, Y, Z; + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); + + for (int i = 0; i < num_points; i++) + { + // Get the points + X = *(mData++); + Y = *(mData++); + Z = *(mData++); + + double x; + double y; + + // if depth is 0 the projection is different + if (Z != 0) + { + x = ((X * fx / Z) + cx); + y = ((Y * fy / Z) + cy); + } + else + { + x = X; + y = Y; + } + + // Project and store in dest matrix + (*projected++) = x; + (*projected++) = y; + } + + } + } #endif \ No newline at end of file diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h index 4d0689b..8c4d452 100644 --- a/lib/local/Utilities/include/VisualizationUtils.h +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -43,8 +43,6 @@ namespace Utilities // TODO draw AU results - // Helper utilities - void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy); // Drawing a bounding box around the face in an image void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); diff --git a/lib/local/Utilities/include/Visualizer.h b/lib/local/Utilities/include/Visualizer.h index 5463da2..3a5d695 100644 --- a/lib/local/Utilities/include/Visualizer.h +++ b/lib/local/Utilities/include/Visualizer.h @@ -56,7 +56,7 @@ namespace Utilities Visualizer(std::vector arguments); Visualizer(bool vis_track, bool vis_hog, bool vis_align); - ~Visualizer(); + //~Visualizer(); // TODO copy, assignment and move operators? Do not allow diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 0deac67..1675164 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -157,7 +157,7 @@ void RecorderOpenFace::WriteObservation() { // As we are writing out the header, work out some things like number of landmarks, names of AUs etc. int num_face_landmarks = landmarks_2D.rows / 2; - int num_eye_landmarks = (int)eye_landmarks.size(); + int num_eye_landmarks = (int)eye_landmarks2D.size(); int num_model_modes = pdm_params_local.rows; std::vector au_names_class; @@ -182,7 +182,7 @@ void RecorderOpenFace::WriteObservation() this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose, - gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks, au_intensities, au_occurences); + gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences); if(params.outputHOG()) { diff --git a/lib/local/Utilities/src/VisualizationUtils.cpp b/lib/local/Utilities/src/VisualizationUtils.cpp index 1e56d47..090ab87 100644 --- a/lib/local/Utilities/src/VisualizationUtils.cpp +++ b/lib/local/Utilities/src/VisualizationUtils.cpp @@ -43,46 +43,6 @@ namespace Utilities { - void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) - { - dest = cv::Mat_(mesh.rows, 2, 0.0); - - int num_points = mesh.rows; - - double X, Y, Z; - - - cv::Mat_::const_iterator mData = mesh.begin(); - cv::Mat_::iterator projected = dest.begin(); - - for (int i = 0; i < num_points; i++) - { - // Get the points - X = *(mData++); - Y = *(mData++); - Z = *(mData++); - - double x; - double y; - - // if depth is 0 the projection is different - if (Z != 0) - { - x = ((X * fx / Z) + cx); - y = ((Y * fy / Z) + cy); - } - else - { - x = X; - y = Y; - } - - // Project and store in dest matrix - (*projected++) = x; - (*projected++) = y; - } - - } void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) { diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index 132a3ee..c5b0ded 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -33,6 +33,7 @@ #include "Visualizer.h" #include "VisualizationUtils.h" +#include "RotationHelpers.h" // For drawing on images #include @@ -149,23 +150,22 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) } } -// TODO add 3D eye landmark locations +// Eye gaze infomration drawing, first of eye landmarks then of gaze void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d) { - // TODO actual drawing, first of eye landmarks then of gaze - if (eye_landmarks.size() > 0) + if (eye_landmarks2d.size() > 0) { - // FIrst draw the eye region landmarks - for (size_t i = 0; i < eye_landmarks.size(); ++i) + // First draw the eye region landmarks + for (size_t i = 0; i < eye_landmarks2d.size(); ++i) { - cv::Point featurePoint(cvRound(eye_landmarks[i].x * (double)draw_multiplier), eye_landmarks[i].y * (double)draw_multiplier)); + cv::Point featurePoint(cvRound(eye_landmarks2d[i].x * (double)draw_multiplier), cvRound(eye_landmarks2d[i].y * (double)draw_multiplier)); // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; + int thickness = 1; + int thickness_2 = 1; - int next_point = i + 1; + size_t next_point = i + 1; if (i == 7) next_point = 0; if (i == 19) @@ -173,7 +173,7 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv if (i == 27) next_point = 20; - cv::Point nextFeaturePoint(cvRound(eye_landmarks[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks[next_point].y * (double)draw_multiplier)); + cv::Point nextFeaturePoint(cvRound(eye_landmarks2d[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks2d[next_point].y * (double)draw_multiplier)); if (i < 8 || i > 19) cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); else @@ -184,36 +184,24 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv // Now draw the gaze lines themselves cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); - int part_left = -1; - int part_right = -1; - for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + // Grabbing the pupil location, to draw eye gaze need to know where the pupil is + cv::Point3d pupil_left(0, 0, 0); + cv::Point3d pupil_right(0, 0, 0); + for (size_t i = 0; i < 8; ++i) { - if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0) - { - part_left = i; - } - if (clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) - { - part_right = i; - } + pupil_left = pupil_left + eye_landmarks3d[i]; + pupil_right = pupil_right + eye_landmarks3d[i + eye_landmarks3d.size()]; } - - cv::Mat eyeLdmks3d_left = clnf_model.hierarchical_models[part_left].GetShape(fx, fy, cx, cy); - cv::Point3f pupil_left = GetPupilPosition(eyeLdmks3d_left); - - cv::Mat_ irisLdmks3d_left = eyeLdmks3d_left.rowRange(0, 8); - cv::Point3f pupil_left(cv::mean(irisLdmks3d_left.col(0))[0], cv::mean(irisLdmks3d_left.col(1))[0], cv::mean(irisLdmks3d_left.col(2))[0]); - - cv::Mat eyeLdmks3d_right = clnf_model.hierarchical_models[part_right].GetShape(fx, fy, cx, cy); - cv::Point3f pupil_right = GetPupilPosition(eyeLdmks3d_right); + pupil_left = pupil_left / 8; + pupil_right = pupil_right / 8; std::vector points_left; points_left.push_back(cv::Point3d(pupil_left)); - points_left.push_back(cv::Point3d(pupil_left + gaze_direction0*50.0)); + points_left.push_back(cv::Point3d(pupil_left + cv::Point3d(gaze_direction0)*50.0)); std::vector points_right; points_right.push_back(cv::Point3d(pupil_right)); - points_right.push_back(cv::Point3d(pupil_right + gaze_direction1*50.0)); + points_right.push_back(cv::Point3d(pupil_right + cv::Point3d(gaze_direction1)*50.0)); cv::Mat_ proj_points; cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); @@ -248,4 +236,7 @@ void Visualizer::ShowObservation() } } - +cv::Mat Visualizer::GetVisImage() +{ + return captured_image; +}