diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 04340dc..2d832bd 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -106,37 +106,6 @@ int frame_count = 0; void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy) { - // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised - double detection_certainty = face_model.detection_certainty; - bool detection_success = face_model.detection_success; - - double visualisation_boundary = 0.4; - - // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if (detection_certainty > visualisation_boundary) - { - LandmarkDetector::Draw(captured_image, face_model); - - double vis_certainty = detection_certainty; - if (vis_certainty > 1) - vis_certainty = 1; - - // Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking - vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary); - - // A rough heuristic for box around the face width - int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); - - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar(vis_certainty*255.0, 0, (1-vis_certainty) * 255), thickness, fx, fy, cx, cy); - - if (det_parameters.track_gaze && detection_success && face_model.eye_model) - { - GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); - } - } // Work out the framerate TODO if (frame_count % 10 == 0) @@ -221,7 +190,7 @@ int main (int argc, char **argv) cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization - if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualize_align || visualize_hog) + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, false); face_analyser.GetLatestAlignedFace(sim_warped_img); diff --git a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj index 884d739..059d064 100644 --- a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj +++ b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj @@ -91,7 +91,7 @@ Level3 Disabled true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 @@ -100,7 +100,7 @@ Level3 Disabled true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) AdvancedVectorExtensions @@ -113,7 +113,7 @@ true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) true WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) StreamingSIMDExtensions2 @@ -131,7 +131,7 @@ true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories) true WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) AdvancedVectorExtensions diff --git a/lib/local/GazeAnalyser/src/GazeEstimation.cpp b/lib/local/GazeAnalyser/src/GazeEstimation.cpp index d9456b8..85346fd 100644 --- a/lib/local/GazeAnalyser/src/GazeEstimation.cpp +++ b/lib/local/GazeAnalyser/src/GazeEstimation.cpp @@ -42,6 +42,7 @@ #include "LandmarkDetectorUtils.h" #include "LandmarkDetectorFunc.h" +#include "Utilities.h" using namespace std; @@ -93,7 +94,7 @@ void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po { cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); cv::Vec3d eulerAngles(headPose(3), headPose(4), headPose(5)); - cv::Matx33d rotMat = LandmarkDetector::Euler2RotationMatrix(eulerAngles); + cv::Matx33d rotMat = Utilities::Euler2RotationMatrix(eulerAngles); int part = -1; for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 4bddb8e..458a6ec 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -51,10 +51,6 @@ using namespace std; namespace LandmarkDetector { -// For subpixel accuracy drawing -const int draw_shiftbits = 4; -const int draw_multiplier = 1 << 4; - // Useful utility for creating directories for storing the output files void create_directory_from_file(string output_path) @@ -802,6 +798,32 @@ vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model) return to_return; } +// Computing the 3D eye landmarks +vector Calculate3DEyeLandmarks(const CLNF& clnf_model, double fx, double fy, double cx, double cy) +{ + + vector to_return; + // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 || + clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + + auto lmks = clnf_model.hierarchical_models[i].GetShape(fx, fy, cx, cy); + + int num_landmarks = lmks.rows / 3; + + for (int lmk = 0; lmk < num_landmarks; ++lmk) + { + cv::Point3d curr_lmk(lmks.at(lmk), lmks.at(lmk + num_landmarks), lmks.at(lmk + 2 * num_landmarks)); + to_return.push_back(curr_lmk); + } + } + } + return to_return; +} // Computing eye landmarks (to be drawn later or in different interfaces) vector CalculateAllEyeLandmarks(const CLNF& clnf_model) { diff --git a/lib/local/Utilities/include/RecorderCSV.h b/lib/local/Utilities/include/RecorderCSV.h index 1f5754a..743c402 100644 --- a/lib/local/Utilities/include/RecorderCSV.h +++ b/lib/local/Utilities/include/RecorderCSV.h @@ -65,7 +65,7 @@ namespace Utilities void WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence, const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Mat_& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate, - const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks, + const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, const std::vector >& au_intensities, const std::vector >& au_occurences); // TODO have set functions? diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h index 3c56de2..70f75ec 100644 --- a/lib/local/Utilities/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -84,7 +84,7 @@ namespace Utilities // Gaze related observations void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, - const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks); + const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2D, const std::vector& eye_landmarks3D); // Face alignment related observations void SetObservationFaceAlign(const cv::Mat& aligned_face); @@ -135,7 +135,8 @@ namespace Utilities cv::Point3f gaze_direction0; cv::Point3f gaze_direction1; cv::Vec2d gaze_angle; - std::vector eye_landmarks; + std::vector eye_landmarks2D; + std::vector eye_landmarks3D; int observation_count; diff --git a/lib/local/Utilities/include/Visualizer.h b/lib/local/Utilities/include/Visualizer.h index cb5f80e..5463da2 100644 --- a/lib/local/Utilities/include/Visualizer.h +++ b/lib/local/Utilities/include/Visualizer.h @@ -72,7 +72,7 @@ namespace Utilities void SetObservationPose(const cv::Vec6d& pose, double confidence); // Gaze related observations - void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks); + void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks, const std::vector& eye_landmarks3d); // Face alignment related observations void SetObservationFaceAlign(const cv::Mat& aligned_face); @@ -83,14 +83,14 @@ namespace Utilities void ShowObservation(); cv::Mat GetVisImage(); - - private: // Keeping track of what we're visualizing bool vis_track; bool vis_hog; bool vis_align; + private: + // Temporary variables for visualization cv::Mat captured_image; // out canvas cv::Mat tracked_image; diff --git a/lib/local/Utilities/src/RecorderCSV.cpp b/lib/local/Utilities/src/RecorderCSV.cpp index fb7e454..78b7510 100644 --- a/lib/local/Utilities/src/RecorderCSV.cpp +++ b/lib/local/Utilities/src/RecorderCSV.cpp @@ -91,6 +91,19 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp { output_file << ", eye_lmk_y_" << i; } + + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_X_" << i; + } + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_Y_" << i; + } + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_Z_" << i; + } } if (output_pose) @@ -160,7 +173,7 @@ bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool outp // TODO check if the stream is open void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence, const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Mat_& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate, - const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks, + const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, const std::vector >& au_intensities, const std::vector >& au_occurences) { @@ -188,15 +201,31 @@ void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landm output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1]; // Output the 2D eye landmarks - for (auto eye_lmk : eye_landmarks) + for (auto eye_lmk : eye_landmarks2d) { output_file << ", " << eye_lmk.x; } - for (auto eye_lmk : eye_landmarks) + for (auto eye_lmk : eye_landmarks2d) { output_file << ", " << eye_lmk.y; } + + // Output the 3D eye landmarks + for (auto eye_lmk : eye_landmarks3d) + { + output_file << ", " << eye_lmk.x; + } + + for (auto eye_lmk : eye_landmarks3d) + { + output_file << ", " << eye_lmk.y; + } + + for (auto eye_lmk : eye_landmarks3d) + { + output_file << ", " << eye_lmk.z; + } } // Output the estimated head pose diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index aab0662..0deac67 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -259,12 +259,13 @@ void RecorderOpenFace::SetObservationActionUnits(const std::vector& eye_landmarks) + const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2D, const std::vector& eye_landmarks3D) { this->gaze_direction0 = gaze_direction0; this->gaze_direction1 = gaze_direction1; this->gaze_angle = gaze_angle; - this->eye_landmarks = eye_landmarks; + this->eye_landmarks2D = eye_landmarks2D; + this->eye_landmarks3D = eye_landmarks3D; } RecorderOpenFace::~RecorderOpenFace() diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index c8ea299..132a3ee 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -34,8 +34,15 @@ #include "Visualizer.h" #include "VisualizationUtils.h" +// For drawing on images +#include + using namespace Utilities; +// For subpixel accuracy drawing +const int draw_shiftbits = 4; +const int draw_multiplier = 1 << 4; + Visualizer::Visualizer(std::vector arguments) { // By default not visualizing anything @@ -97,7 +104,26 @@ void Visualizer::SetObservationHOG(const cv::Mat_& hog_descriptor, int n void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, bool success, const cv::Mat_& visibilities) { - DrawLandmarkDetResults(captured_image, landmarks_2D, visibilities); + + // Draw 2D landmarks on the image + int n = landmarks_2D.rows / 2; + + // Drawing feature points + for (int i = 0; i < n; ++i) + { + if (visibilities.empty() || visibilities.at(i)) + { + cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (double)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = (int)std::ceil(3.0* ((double)captured_image.cols) / 640.0); + int thickness_2 = (int)std::ceil(1.0* ((double)captured_image.cols) / 640.0); + + cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); + cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + + } + } } void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) @@ -123,18 +149,103 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) } } - -void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, - const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks) +// TODO add 3D eye landmark locations +void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d) { - // TODO actual drawing + // TODO actual drawing, first of eye landmarks then of gaze - if (det_parameters.track_gaze && detection_success && face_model.eye_model) + if (eye_landmarks.size() > 0) { - GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); + // FIrst draw the eye region landmarks + for (size_t i = 0; i < eye_landmarks.size(); ++i) + { + cv::Point featurePoint(cvRound(eye_landmarks[i].x * (double)draw_multiplier), eye_landmarks[i].y * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = 1.0; + int thickness_2 = 1.0; + + int next_point = i + 1; + if (i == 7) + next_point = 0; + if (i == 19) + next_point = 8; + if (i == 27) + next_point = 20; + + cv::Point nextFeaturePoint(cvRound(eye_landmarks[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks[next_point].y * (double)draw_multiplier)); + if (i < 8 || i > 19) + cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + else + cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); + + } + + // Now draw the gaze lines themselves + cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); + + int part_left = -1; + int part_right = -1; + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0) + { + part_left = i; + } + if (clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + part_right = i; + } + } + + cv::Mat eyeLdmks3d_left = clnf_model.hierarchical_models[part_left].GetShape(fx, fy, cx, cy); + cv::Point3f pupil_left = GetPupilPosition(eyeLdmks3d_left); + + cv::Mat_ irisLdmks3d_left = eyeLdmks3d_left.rowRange(0, 8); + cv::Point3f pupil_left(cv::mean(irisLdmks3d_left.col(0))[0], cv::mean(irisLdmks3d_left.col(1))[0], cv::mean(irisLdmks3d_left.col(2))[0]); + + cv::Mat eyeLdmks3d_right = clnf_model.hierarchical_models[part_right].GetShape(fx, fy, cx, cy); + cv::Point3f pupil_right = GetPupilPosition(eyeLdmks3d_right); + + std::vector points_left; + points_left.push_back(cv::Point3d(pupil_left)); + points_left.push_back(cv::Point3d(pupil_left + gaze_direction0*50.0)); + + std::vector points_right; + points_right.push_back(cv::Point3d(pupil_right)); + points_right.push_back(cv::Point3d(pupil_right + gaze_direction1*50.0)); + + cv::Mat_ proj_points; + cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); + Project(proj_points, mesh_0, fx, fy, cx, cy); + cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (double)draw_multiplier), cvRound(proj_points.at(0, 1) * (double)draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * (double)draw_multiplier), cvRound(proj_points.at(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); + + cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); + Project(proj_points, mesh_1, fx, fy, cx, cy); + cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (double)draw_multiplier), cvRound(proj_points.at(0, 1) * (double)draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * (double)draw_multiplier), cvRound(proj_points.at(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); + + } + +} + +void Visualizer::ShowObservation() +{ + if (vis_track) + { + cv::namedWindow("tracking_result", 1); + cv::imshow("tracking_result", captured_image); + cv::waitKey(1); + } + if (vis_align) + { + cv::imshow("sim_warp", aligned_face_image); + } + if (vis_hog) + { + cv::imshow("hog", hog_image); } } - -