diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 42acc23..9448cf3 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -192,7 +192,7 @@ int main (int argc, char **argv) // Displaying the tracking visualizations visualizer.SetObservationFaceAlign(sim_warped_img); visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); - visualizer.SetObservationLandmarks(face_model.detected_landmarks, 1.0); // Set confidence to high to make sure we always visualize + visualizer.SetObservationLandmarks(face_model.detected_landmarks, 1.0, face_model.GetVisibilities()); // Set confidence to high to make sure we always visualize visualizer.SetObservationPose(pose_estimate, 1.0); visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy), face_model.detection_certainty); diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 55c90d0..d1bf5ec 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -164,7 +164,7 @@ int main (int argc, char **argv) // Displaying the tracking visualizations visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); visualizer.SetFps(fps_tracker.GetFPS()); diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 3807cac..a3c9e04 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -194,7 +194,7 @@ int main (int argc, char **argv) visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); visualizer.SetObservationFaceAlign(sim_warped_img); visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); - visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); visualizer.SetFps(fps_tracker.GetFPS()); diff --git a/gui/HeadPose-live/MainWindow.xaml.cs b/gui/HeadPose-live/MainWindow.xaml.cs index da63b89..950901f 100644 --- a/gui/HeadPose-live/MainWindow.xaml.cs +++ b/gui/HeadPose-live/MainWindow.xaml.cs @@ -417,7 +417,7 @@ namespace HeadPoseLive List landmarks = new List(); List> gaze_lines = null; Tuple gaze_angle = new Tuple(0, 0); - double scale = 0; + double scale = face_model.GetRigidParams()[0]; if (detectionSucceeding) { @@ -428,9 +428,7 @@ namespace HeadPoseLive eye_landmarks = face_model.CalculateVisibleEyeLandmarks(); - scale = face_model.GetRigidParams()[0]; - - gaze_lines = gaze_analyser.CalculateGazeLines(scale, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); + gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); gaze_angle = gaze_analyser.GetGazeAngle(); lines = face_model.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); diff --git a/gui/OpenFaceDemo/MainWindow.xaml.cs b/gui/OpenFaceDemo/MainWindow.xaml.cs index 82147ee..aeeb6fd 100644 --- a/gui/OpenFaceDemo/MainWindow.xaml.cs +++ b/gui/OpenFaceDemo/MainWindow.xaml.cs @@ -251,7 +251,7 @@ namespace OpenFaceDemo landmarks = landmark_detector.CalculateVisibleLandmarks(); eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks(); lines = landmark_detector.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); - gaze_lines = gaze_analyser.CalculateGazeLines(scale, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); + gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); } // Visualisation diff --git a/gui/OpenFaceOffline/MainWindow.xaml b/gui/OpenFaceOffline/MainWindow.xaml index 1403b8d..c768161 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml +++ b/gui/OpenFaceOffline/MainWindow.xaml @@ -64,7 +64,7 @@ - + diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index 1e18dbe..310a827 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -224,7 +224,7 @@ namespace OpenFaceOffline gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details - VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), detection_succeeding, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); + VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp()); @@ -237,8 +237,6 @@ namespace OpenFaceOffline if (skip_frames > 0) skip_frames--; - latest_img = null; - frame = new RawImage(reader.GetNextImage()); gray_frame = new RawImage(reader.GetCurrentFrameGray()); @@ -330,15 +328,13 @@ namespace OpenFaceOffline gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details - VisualizeFeatures(frame, visualizer_of, landmarks, detection_succeeding, i == 0, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); + VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0); } - latest_img = null; - frame = new RawImage(reader.GetNextImage()); gray_frame = new RawImage(reader.GetCurrentFrameGray()); @@ -398,7 +394,7 @@ namespace OpenFaceOffline } - private void VisualizeFeatures(RawImage frame, Visualizer visualizer, List> landmarks, bool detection_succeeding, + private void VisualizeFeatures(RawImage frame, Visualizer visualizer, List> landmarks, List visibilities, bool detection_succeeding, bool new_image, float fx, float fy, float cx, float cy, double progress) { @@ -418,27 +414,24 @@ namespace OpenFaceOffline else if (confidence > 1) confidence = 1; - double scale = 0; + double scale = landmark_detector.GetRigidParams()[0]; // Helps with recording and showing the visualizations - if(new_image) + if (new_image) { visualizer.SetImage(frame, fx, fy, cx, cy); } visualizer.SetObservationHOG(face_analyser.GetLatestHOGFeature(), face_analyser.GetHOGRows(), face_analyser.GetHOGCols()); - visualizer.SetObservationLandmarks(landmarks, confidence); // Set confidence to high to make sure we always visualize + visualizer.SetObservationLandmarks(landmarks, confidence, visibilities); visualizer.SetObservationPose(pose, confidence); visualizer.SetObservationGaze(gaze_analyser.GetGazeCamera().Item1, gaze_analyser.GetGazeCamera().Item2, landmark_detector.CalculateAllEyeLandmarks(), landmark_detector.CalculateAllEyeLandmarks3D(fx, fy, cx, cy), confidence); if (detection_succeeding) - { - + { eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks(); lines = landmark_detector.CalculateBox(fx, fy, cx, cy); - scale = landmark_detector.GetRigidParams()[0]; - - gaze_lines = gaze_analyser.CalculateGazeLines(scale, fx, fy, cx, cy); + gaze_lines = gaze_analyser.CalculateGazeLines(fx, fy, cx, cy); gaze_angle = gaze_analyser.GetGazeAngle(); } @@ -490,25 +483,26 @@ namespace OpenFaceOffline if (ShowTrackedVideo) { - if (latest_img == null) + if (new_image) { latest_img = frame.CreateWriteableBitmap(); } frame.UpdateWriteableBitmap(latest_img); - - video.Source = latest_img; - video.Confidence = confidence; - video.FPS = processing_fps.GetFPS(); - video.Progress = progress; - video.FaceScale = scale; + + overlay_image.Source = latest_img; + overlay_image.Confidence = confidence; + overlay_image.FPS = processing_fps.GetFPS(); + overlay_image.Progress = progress; + overlay_image.FaceScale = scale; if (!detection_succeeding) { - video.OverlayLines.Clear(); - video.OverlayPoints.Clear(); - video.OverlayEyePoints.Clear(); - video.GazeLines.Clear(); + overlay_image.OverlayLines.Clear(); + overlay_image.OverlayPoints.Clear(); + overlay_image.OverlayPointsVisibility.Clear(); + overlay_image.OverlayEyePoints.Clear(); + overlay_image.GazeLines.Clear(); } else { @@ -528,18 +522,20 @@ namespace OpenFaceOffline if (new_image) { - video.OverlayLines = lines; - video.OverlayPoints = landmark_points; - video.OverlayEyePoints = eye_landmark_points; - video.GazeLines = gaze_lines; + overlay_image.OverlayLines = lines; + overlay_image.OverlayPoints = landmark_points; + overlay_image.OverlayPointsVisibility = visibilities; + overlay_image.OverlayEyePoints = eye_landmark_points; + overlay_image.GazeLines = gaze_lines; } else { // In case of multiple faces just add them to the existing drawing list - video.OverlayLines.AddRange(lines.GetRange(0, lines.Count)); - video.OverlayPoints.AddRange(landmark_points.GetRange(0, landmark_points.Count)); - video.OverlayEyePoints.AddRange(eye_landmark_points.GetRange(0, eye_landmark_points.Count)); - video.GazeLines.AddRange(gaze_lines.GetRange(0, gaze_lines.Count)); + overlay_image.OverlayLines.AddRange(lines.GetRange(0, lines.Count)); + overlay_image.OverlayPoints.AddRange(landmark_points.GetRange(0, landmark_points.Count)); + overlay_image.OverlayPointsVisibility.AddRange(visibilities.GetRange(0, visibilities.Count)); + overlay_image.OverlayEyePoints.AddRange(eye_landmark_points.GetRange(0, eye_landmark_points.Count)); + overlay_image.GazeLines.AddRange(gaze_lines.GetRange(0, gaze_lines.Count)); } } } @@ -626,7 +622,7 @@ namespace OpenFaceOffline NextFrameButton.IsEnabled = false; // Clean up the interface itself - video.Source = null; + overlay_image.Source = null; auClassGraph.Update(new Dictionary()); auRegGraph.Update(new Dictionary()); diff --git a/gui/OpenFaceOffline/UI_items/OverlayImage.xaml.cs b/gui/OpenFaceOffline/UI_items/OverlayImage.xaml.cs index 16cce83..ee63fbd 100644 --- a/gui/OpenFaceOffline/UI_items/OverlayImage.xaml.cs +++ b/gui/OpenFaceOffline/UI_items/OverlayImage.xaml.cs @@ -75,6 +75,7 @@ namespace OpenFaceOffline InitializeComponent(); OverlayLines = new List>(); OverlayPoints = new List(); + OverlayPointsVisibility = new List(); OverlayEyePoints = new List(); GazeLines = new List>(); @@ -91,6 +92,9 @@ namespace OpenFaceOffline if (OverlayPoints == null) OverlayPoints = new List(); + if (OverlayPointsVisibility == null) + OverlayPointsVisibility = new List(); + if (OverlayEyePoints == null) OverlayEyePoints = new List(); @@ -122,17 +126,29 @@ namespace OpenFaceOffline var p1 = new Point(ActualWidth * line.Item1.X / width, ActualHeight * line.Item1.Y / height); var p2 = new Point(ActualWidth * line.Item2.X / width, ActualHeight * line.Item2.Y / height); - dc.DrawLine(new Pen(new SolidColorBrush(Color.FromArgb(200, (byte)(240), (byte)(30), (byte)100)), 3.0 * scaling_p), p1, p2); + var dir = p2 - p1; + p2 = p1 + dir * scaling_p * 2; + dc.DrawLine(new Pen(new SolidColorBrush(Color.FromArgb(200, (byte)(240), (byte)(30), (byte)100)), 6.0 * scaling_p), p1, p2); } - foreach (var p in OverlayPoints) + for (int i = 0; i < OverlayPoints.Count; ++i) { + var p = OverlayPoints[i]; var q = new Point(ActualWidth * p.X / width, ActualHeight * p.Y / height); - dc.DrawEllipse(new SolidColorBrush(Color.FromArgb((byte)(230 * Confidence), 255, 50, 50)), null, q, 2.75 * scaling_p, 2.75 * scaling_p); - dc.DrawEllipse(new SolidColorBrush(Color.FromArgb((byte)(230 * Confidence), 255, 255, 100)), null, q, 1.75 * scaling_p, 1.75 * scaling_p); + if(OverlayPointsVisibility.Count == 0 || OverlayPointsVisibility[i]) + { + dc.DrawEllipse(new SolidColorBrush(Color.FromArgb((byte)(230 * Confidence), 255, 50, 50)), null, q, 2.75 * scaling_p, 3.0 * scaling_p); + dc.DrawEllipse(new SolidColorBrush(Color.FromArgb((byte)(230 * Confidence), 255, 255, 100)), null, q, 1.75 * scaling_p, 2.0 * scaling_p); + } + else + { + // Draw fainter if landmark not visible + dc.DrawEllipse(new SolidColorBrush(Color.FromArgb((byte)(125 * Confidence), 255, 50, 50)), null, q, 2.75 * scaling_p, 3.0 * scaling_p); + dc.DrawEllipse(new SolidColorBrush(Color.FromArgb((byte)(125 * Confidence), 255, 255, 100)), null, q, 1.75 * scaling_p, 2.0 * scaling_p); + } } for (int id = 0; id < OverlayEyePoints.Count; id++) @@ -199,6 +215,7 @@ namespace OpenFaceOffline public List> OverlayLines { get; set; } public List> GazeLines { get; set; } public List OverlayPoints { get; set; } + public List OverlayPointsVisibility { get; set; } public List OverlayEyePoints { get; set; } public double Confidence { get; set; } public double FPS { get; set; } diff --git a/lib/local/CppInerop/GazeAnalyserInterop.h b/lib/local/CppInerop/GazeAnalyserInterop.h index f81e16d..6629202 100644 --- a/lib/local/CppInerop/GazeAnalyserInterop.h +++ b/lib/local/CppInerop/GazeAnalyserInterop.h @@ -152,18 +152,18 @@ namespace GazeAnalyser_Interop { return gaze_angle; } - System::Collections::Generic::List^>^ CalculateGazeLines(double scale, float fx, float fy, float cx, float cy) + System::Collections::Generic::List^>^ CalculateGazeLines(float fx, float fy, float cx, float cy) { cv::Mat_ cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); vector points_left; points_left.push_back(cv::Point3f(*pupil_left)); - points_left.push_back(cv::Point3f(*pupil_left + *gazeDirection0 * 40.0 * scale)); + points_left.push_back(cv::Point3f(*pupil_left + *gazeDirection0 * 40.0)); vector points_right; points_right.push_back(cv::Point3f(*pupil_right)); - points_right.push_back(cv::Point3f(*pupil_right + *gazeDirection1 * 40.0 * scale)); + points_right.push_back(cv::Point3f(*pupil_right + *gazeDirection1 * 40.0)); // Perform manual projection of points vector imagePoints_left; diff --git a/lib/local/CppInerop/LandmarkDetectorInterop.h b/lib/local/CppInerop/LandmarkDetectorInterop.h index 8a65fec..b098e68 100644 --- a/lib/local/CppInerop/LandmarkDetectorInterop.h +++ b/lib/local/CppInerop/LandmarkDetectorInterop.h @@ -227,6 +227,19 @@ namespace CppInterop { } } + // Get the mask of which landmarks are currently visible (not self-occluded) + List^ GetVisibilities() + { + cv::Mat_ vis = clnf->GetVisibilities(); + List^ visibilities = gcnew List(); + + for (auto vis_it = vis.begin(); vis_it != vis.end(); vis_it++) + { + visibilities->Add(*vis_it != 0); + } + return visibilities; + } + List^>^ CalculateVisibleLandmarks() { vector vecLandmarks = ::LandmarkDetector::CalculateVisibleLandmarks(*clnf); diff --git a/lib/local/CppInerop/VisualizerInterop.h b/lib/local/CppInerop/VisualizerInterop.h index 512d4a5..db8b2ca 100644 --- a/lib/local/CppInerop/VisualizerInterop.h +++ b/lib/local/CppInerop/VisualizerInterop.h @@ -100,7 +100,7 @@ namespace UtilitiesOF { m_visualizer->SetObservationHOG(observation_HOG->Mat, num_cols, num_rows); } - void SetObservationLandmarks(List^>^ landmarks_2D, double confidence) + void SetObservationLandmarks(List^>^ landmarks_2D, double confidence, List^ visibilities) { // Construct an OpenCV matrix from the landmarks cv::Mat_ landmarks_2D_mat(landmarks_2D->Count * 2, 1, 0.0); @@ -109,8 +109,27 @@ namespace UtilitiesOF { landmarks_2D_mat.at(i, 0) = landmarks_2D[i]->Item1; landmarks_2D_mat.at(i + landmarks_2D->Count, 0) = landmarks_2D[i]->Item2; } - // TODO add visibilities - m_visualizer->SetObservationLandmarks(landmarks_2D_mat, confidence); + + // Construct an OpenCV matrix from the landmarks + cv::Mat_ visibilities_cv(visibilities->Count, 1, 0); + for (int i = 0; i < visibilities->Count; ++i) + { + if (visibilities[i]) + { + visibilities_cv.at(i, 0) = 1; + } + else + { + visibilities_cv.at(i, 0) = 0; + } + } + + m_visualizer->SetObservationLandmarks(landmarks_2D_mat, confidence, visibilities_cv); + } + + void SetObservationLandmarks(List^>^ landmarks_2D, double confidence) + { + SetObservationLandmarks(landmarks_2D, confidence, gcnew List()); } void SetImage(OpenCVWrappers::RawImage^ canvas, float fx, float fy, float cx, float cy) diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index 04a1550..42ef229 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -159,6 +159,9 @@ public: // A utility bounding box function cv::Rect_ GetBoundingBox() const; + // Get the currently non-self occluded landmarks + cv::Mat_ GetVisibilities() const; + // Reset the model (useful if we want to completelly reinitialise, or we want to track another video) void Reset(); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 5e3fecf..381f766 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -1146,6 +1146,17 @@ cv::Mat_ CLNF::GetShape(double fx, double fy, double cx, double cy) cons } +cv::Mat_ CLNF::GetVisibilities() const +{ + // Get the view of the largest scale + int scale = patch_experts.visibilities.size() - 1; + int view_id = patch_experts.GetViewIdx(params_global, scale); + + cv::Mat_ visibilities_to_ret = this->patch_experts.visibilities[scale][view_id].clone(); + return visibilities_to_ret; +} + + // A utility bounding box function cv::Rect_ CLNF::GetBoundingBox() const { diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index 69313b4..16f1ba5 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -157,6 +157,19 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, d cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); } + else + { + // Draw a fainter point if the landmark is self occluded + cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (double)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = (int)std::ceil(2.5* ((double)captured_image.cols) / 640.0); + int thickness_2 = (int)std::ceil(1.0* ((double)captured_image.cols) / 640.0); + + cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 155), thickness, CV_AA, draw_shiftbits); + cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(155, 0, 0), thickness_2, CV_AA, draw_shiftbits); + + } } } }