From ea0f6581908558735a4cd66dbe8dc71637a13edd Mon Sep 17 00:00:00 2001
From: Tadas Baltrusaitis <tadyla@gmail.com>
Date: Sat, 25 Nov 2017 10:12:34 +0000
Subject: [PATCH] Dealing better with non-tracked frames.

---
 exe/FeatureExtraction/FeatureExtraction.cpp   |  2 +-
 .../src/LandmarkDetectorModel.cpp             | 56 ++++++++++---------
 2 files changed, 32 insertions(+), 26 deletions(-)

diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp
index fe7b2bb..97e28c0 100644
--- a/exe/FeatureExtraction/FeatureExtraction.cpp
+++ b/exe/FeatureExtraction/FeatureExtraction.cpp
@@ -162,7 +162,7 @@ int main (int argc, char **argv)
 			bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters);
 
 			// Gaze tracking, absolute gaze direction
-			cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); cv::Vec2d gazeAngle(0, 0);
+			cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0);
 
 			if (detection_success && face_model.eye_model)
 			{
diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp
index 14041c3..958c42f 100644
--- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp
+++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp
@@ -1081,36 +1081,42 @@ cv::Mat_<double> CLNF::GetShape(double fx, double fy, double cx, double cy) cons
 {
 	int n = this->detected_landmarks.rows/2;
 
-	cv::Mat_<double> shape3d(n*3, 1);
+	cv::Mat_<double> outShape(n, 3, 0.0);
 
-	this->pdm.CalcShape3D(shape3d, this->params_local);
-	
-	// Need to rotate the shape to get the actual 3D representation
-	
-	// get the rotation matrix from the euler angles
-	cv::Matx33d R = Utilities::Euler2RotationMatrix(cv::Vec3d(params_global[1], params_global[2], params_global[3]));
-
-	shape3d = shape3d.reshape(1, 3);
-
-	shape3d = shape3d.t() * cv::Mat(R).t();
-	
-	// from the weak perspective model can determine the average depth of the object
-	double Zavg = fx / params_global[0];	
-
-	cv::Mat_<double> outShape(n,3,0.0);
-
-	// this is described in the paper in section 3.4 (equation 10) (of the CLM-Z paper)
-	for(int i = 0; i < n; i++)
+	// If the tracking started (otherwise no point reporting 3D shape)
+	if(this->tracking_initialised)
 	{
-		double Z = Zavg + shape3d.at<double>(i,2);
 
-		double X = Z * ((this->detected_landmarks.at<double>(i) - cx)/fx);
-		double Y = Z * ((this->detected_landmarks.at<double>(i + n) - cy)/fy);
+		cv::Mat_<double> shape3d(n * 3, 1);
 
-		outShape.at<double>(i,0) = (double)X;
-		outShape.at<double>(i,1) = (double)Y;
-		outShape.at<double>(i,2) = (double)Z;
+		this->pdm.CalcShape3D(shape3d, this->params_local);
+	
+		// Need to rotate the shape to get the actual 3D representation
+	
+		// get the rotation matrix from the euler angles
+		cv::Matx33d R = Utilities::Euler2RotationMatrix(cv::Vec3d(params_global[1], params_global[2], params_global[3]));
 
+		shape3d = shape3d.reshape(1, 3);
+
+		shape3d = shape3d.t() * cv::Mat(R).t();
+	
+		// from the weak perspective model can determine the average depth of the object
+		double Zavg = fx / params_global[0];	
+
+
+		// this is described in the paper in section 3.4 (equation 10) (of the CLM-Z paper)
+		for(int i = 0; i < n; i++)
+		{
+			double Z = Zavg + shape3d.at<double>(i,2);
+
+			double X = Z * ((this->detected_landmarks.at<double>(i) - cx)/fx);
+			double Y = Z * ((this->detected_landmarks.at<double>(i + n) - cy)/fy);
+
+			outShape.at<double>(i,0) = (double)X;
+			outShape.at<double>(i,1) = (double)Y;
+			outShape.at<double>(i,2) = (double)Z;
+
+		}
 	}
 
 	// The format is 3 rows - n cols