More changes to bring up to speed for GUI:

- Outputting gaze angle
- Outputting eye landmarks

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -204,13 +204,13 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
 
 void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
-	int num_landmarks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg);
+	int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg);
 
 // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow)
 void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
 	const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success,
-	cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
+	cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
 	const FaceAnalysis::FaceAnalyser& face_analyser);
 
 int main (int argc, char **argv)
@@ -466,7 +466,9 @@ int main (int argc, char **argv)
 		if (!output_files.empty())
 		{
 			output_file.open(output_files[f_n], ios_base::out);
-			prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
+			int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size();
+		
+			prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames());
 		}
 
 		// Saving the HOG features
@@ -548,11 +550,13 @@ int main (int argc, char **argv)
 			// Gaze tracking, absolute gaze direction
 			cv::Point3f gazeDirection0(0, 0, -1);
 			cv::Point3f gazeDirection1(0, 0, -1);
+			cv::Vec2d gazeAngle(0, 0);
 
 			if (det_parameters.track_gaze && detection_success && face_model.eye_model)
 			{
 				FaceAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true);
 				FaceAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false);
+				gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1);
 			}
 
 			// Do face alignment
@@ -635,7 +639,7 @@ int main (int argc, char **argv)
 
 			// Output the landmarks, pose, gaze, parameters and AUs
 			outputAllFeatures(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze,
-				face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1,
+				face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, gazeAngle,
 				pose_estimate, fx, fy, cx, cy, face_analyser);
 
 			// output the tracked video
@@ -724,14 +728,23 @@ int main (int argc, char **argv)
 
 void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
-	int num_landmarks, int num_model_modes, vector<string> au_names_class, vector<string> au_names_reg)
+	int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector<string> au_names_class, vector<string> au_names_reg)
 {
 
 	*output_file << "frame, timestamp, confidence, success";
 
 	if (output_gaze)
 	{
-		*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z";
+		*output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
+
+		for (int i = 0; i < num_eye_landmarks; ++i)
+		{
+			*output_file << ", eye_lmk_x_" << i;
+		}
+		for (int i = 0; i < num_eye_landmarks; ++i)
+		{
+			*output_file << ", eye_lmk_y_" << i;
+		}
 	}
 
 	if (output_pose)
@@ -741,11 +754,11 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
 
 	if (output_2D_landmarks)
 	{
-		for (int i = 0; i < num_landmarks; ++i)
+		for (int i = 0; i < num_face_landmarks; ++i)
 		{
 			*output_file << ", x_" << i;
 		}
-		for (int i = 0; i < num_landmarks; ++i)
+		for (int i = 0; i < num_face_landmarks; ++i)
 		{
 			*output_file << ", y_" << i;
 		}
@@ -753,15 +766,15 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
 
 	if (output_3D_landmarks)
 	{
-		for (int i = 0; i < num_landmarks; ++i)
+		for (int i = 0; i < num_face_landmarks; ++i)
 		{
 			*output_file << ", X_" << i;
 		}
-		for (int i = 0; i < num_landmarks; ++i)
+		for (int i = 0; i < num_face_landmarks; ++i)
 		{
 			*output_file << ", Y_" << i;
 		}
-		for (int i = 0; i < num_landmarks; ++i)
+		for (int i = 0; i < num_face_landmarks; ++i)
 		{
 			*output_file << ", Z_" << i;
 		}
@@ -800,7 +813,7 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo
 void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks,
 	bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
 	const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success,
-	cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
+	cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
 	const FaceAnalysis::FaceAnalyser& face_analyser)
 {
 
@@ -813,6 +826,21 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
 	{
 		*output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z
 			<< ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z;
+
+		// Output gaze angle (same format as head pose angle)
+		*output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1];
+
+		// Output eye landmarks
+		std::vector<cv::Point2d> eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(face_model);
+
+		for (size_t i = 0; i < eye_landmark_points.size(); ++i)
+		{
+			*output_file << ", " << eye_landmark_points[i].x;
+		}
+		for (size_t i = 0; i < eye_landmark_points.size(); ++i)
+		{
+			*output_file << ", " << eye_landmark_points[i].y;
+		}
 	}
 
 	// Output the estimated head pose

lib/local/FaceAnalyser/include/GazeEstimation.h

@@ -44,5 +44,9 @@ namespace FaceAnalysis
 	void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye);
 	void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy);
 
+	// Getting the gaze angle in radians with respect to the world coordinates (camera plane), when looking ahead straight at camera plane the gaze angle will be (0,0)
+	cv::Vec2d GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2);
+
+
 }
 #endif

lib/local/FaceAnalyser/src/GazeEstimation.cpp

@@ -133,6 +133,17 @@ void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po
 	gaze_absolute = gazeVecAxis / norm(gazeVecAxis);
 }
 
+cv::Vec2d FaceAnalysis::GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2)
+{
+
+	cv::Point3f gaze_vector = (gaze_vector_1 + gaze_vector_2) / 2;
+
+	double x_angle = atan2(gaze_vector.x, -gaze_vector.z);
+	double y_angle = atan2(gaze_vector.y, -gaze_vector.z);
+
+	return cv::Vec2d(x_angle, y_angle);
+
+}
 
 void FaceAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy)
 {

lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h

@@ -89,8 +89,13 @@ namespace LandmarkDetector
 	vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy);
 	void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar color, int thickness);
 
-	vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_<int>& visibilities);
+	vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);
-	vector<cv::Point2d> CalculateLandmarks(CLNF& clnf_model);
+	vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model);
+	vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model);
+
+	vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D);
+	vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model);
+	vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model);
 	void DrawLandmarks(cv::Mat img, vector<cv::Point> landmarks);
 
 	void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);

lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp

@@ -970,16 +970,16 @@ void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar
 }
 
 // Computing landmarks (to be drawn later possibly)
-vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_<int>& visibilities)
+vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
 {
-	int n = shape2D.rows/2;
+	int n = shape2D.rows / 2;
 	vector<cv::Point2d> landmarks;
 
-	for( int i = 0; i < n; ++i)
+	for (int i = 0; i < n; ++i)
 	{
-		if(visibilities.at<int>(i))
+		if (visibilities.at<int>(i))
 		{
-			cv::Point2d featurePoint(shape2D.at<double>(i), shape2D.at<double>(i +n));
+			cv::Point2d featurePoint(shape2D.at<double>(i), shape2D.at<double>(i + n));
 
 			landmarks.push_back(featurePoint);
 		}
@@ -989,27 +989,27 @@ vector<cv::Point2d> CalculateLandmarks(const cv::Mat_<double>& shape2D, cv::Mat_
 }
 
 // Computing landmarks (to be drawn later possibly)
-vector<cv::Point2d> CalculateLandmarks(cv::Mat img, const cv::Mat_<double>& shape2D)
+vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D)
 {
 
 	int n;
 	vector<cv::Point2d> landmarks;
 
-	if(shape2D.cols == 2)
+	if (shape2D.cols == 2)
 	{
 		n = shape2D.rows;
 	}
-	else if(shape2D.cols == 1)
+	else if (shape2D.cols == 1)
 	{
-		n = shape2D.rows/2;
+		n = shape2D.rows / 2;
 	}
 
-	for( int i = 0; i < n; ++i)
+	for (int i = 0; i < n; ++i)
 	{
 		cv::Point2d featurePoint;
-		if(shape2D.cols == 1)
+		if (shape2D.cols == 1)
 		{
-			featurePoint = cv::Point2d(shape2D.at<double>(i), shape2D.at<double>(i +n));
+			featurePoint = cv::Point2d(shape2D.at<double>(i), shape2D.at<double>(i + n));
 		}
 		else
 		{
@@ -1023,16 +1023,74 @@ vector<cv::Point2d> CalculateLandmarks(cv::Mat img, const cv::Mat_<double>& shap
 }
 
 // Computing landmarks (to be drawn later possibly)
-vector<cv::Point2d> CalculateLandmarks(CLNF& clnf_model)
+vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model)
+{
+	return CalculateAllLandmarks(clnf_model.detected_landmarks);
+}
+
+// Computing landmarks (to be drawn later possibly)
+vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model)
+{
+	// If the detection was not successful no landmarks are visible
+	if (clnf_model.detection_success)
+	{
+		int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
+		// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
+		return CalculateVisibleLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
+	}
+	else
+	{
+		return vector<cv::Point2d>();
+	}
+}
+
+// Computing eye landmarks (to be drawn later or in different interfaces)
+vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model)
 {
 
-	int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
+	vector<cv::Point2d> to_return;
+	// If the model has hierarchical updates draw those too
+	for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
+	{
 
-	// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
+		if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 ||
-	return CalculateLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
+			clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
+		{
 
+			auto lmks = CalculateVisibleLandmarks(clnf_model.hierarchical_models[i]);
+			for (auto lmk : lmks)
+			{
+				to_return.push_back(lmk);
+			}
+		}
+	}
+	return to_return;
 }
 
+// Computing eye landmarks (to be drawn later or in different interfaces)
+vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model)
+{
+
+	vector<cv::Point2d> to_return;
+	// If the model has hierarchical updates draw those too
+	for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
+	{
+
+		if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 ||
+			clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
+		{
+
+			auto lmks = CalculateAllLandmarks(clnf_model.hierarchical_models[i]);
+			for (auto lmk : lmks)
+			{
+				to_return.push_back(lmk);
+			}
+		}
+	}
+	return to_return;
+}
+
+
 // Drawing landmarks on a face image
 void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
 {

matlab_runners/Demos/feature_extraction_demo_vid.m

@@ -82,10 +82,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_'))
 xs = all_params(valid_frames, landmark_inds_x);
 ys = all_params(valid_frames, landmark_inds_y);
 
+eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_'));
+eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_'));
+
+eye_xs = all_params(valid_frames, eye_landmark_inds_x);
+eye_ys = all_params(valid_frames, eye_landmark_inds_y);
+
 figure
 
 for j = 1:size(xs,1)
     plot(xs(j,:), -ys(j,:), '.');
+    hold on;
+    plot(eye_xs(j,:), -eye_ys(j,:), '.r');
+    hold off;
+    
     xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]);
     ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]);
     xlabel('x (px)');
@@ -141,21 +151,16 @@ title('Pose (rotation and translation)');
 xlabel('Time (s)');
 
 %% Demo gaze
-gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_'));
+gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle'));
 
 % Read gaze (x,y,z) for one eye and (x,y,z) for another
 gaze = all_params(valid_frames, gaze_inds);
 
-% only picking left, right and up down views for visualisation
-gaze = (gaze(:,[1,2]) + gaze(:,[4,5]))/2;
-gaze(:,1) = smooth(gaze(:,1));
-gaze(:,2) = smooth(gaze(:,2));
-
 plot(time, gaze(:,1), 'DisplayName', 'Left - right');
 hold on;
 plot(time, gaze(:,2), 'DisplayName', 'Up - down');
 xlabel('Time(s)') % x-axis label
-ylabel('Gaze vector size') % y-axis label
+ylabel('Angle radians') % y-axis label
 legend('show');
 hold off;