From dba96e15519558b5c1603012bcf208e626607c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9verin=20Lemaignan?= Date: Mon, 12 Mar 2018 17:34:04 +0000 Subject: [PATCH] Added support for AUs visualization to FeatureExtractor --- exe/FeatureExtraction/FeatureExtraction.cpp | 1 + lib/local/Utilities/include/Visualizer.h | 8 +- lib/local/Utilities/src/Visualizer.cpp | 83 +++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index b074d9f..94caecb 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -205,6 +205,7 @@ int main (int argc, char **argv) visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); + visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); visualizer.SetFps(fps_tracker.GetFPS()); // detect key presses diff --git a/lib/local/Utilities/include/Visualizer.h b/lib/local/Utilities/include/Visualizer.h index 813a5e4..bfe6ff2 100644 --- a/lib/local/Utilities/include/Visualizer.h +++ b/lib/local/Utilities/include/Visualizer.h @@ -66,7 +66,9 @@ namespace Utilities // Pose related observations void SetObservationPose(const cv::Vec6d& pose, double confidence); - + + void SetObservationActionUnits(const std::vector >& au_intensities, const std::vector >& au_occurences); + // Gaze related observations void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector& eye_landmarks, const std::vector& eye_landmarks3d, double confidence); @@ -88,6 +90,7 @@ namespace Utilities bool vis_track; bool vis_hog; bool vis_align; + bool vis_aus; // Can be adjusted to show less confident frames double visualisation_boundary = 0.4; @@ -99,10 +102,11 @@ namespace Utilities cv::Mat tracked_image; cv::Mat hog_image; cv::Mat aligned_face_image; + cv::Mat action_units_image; // Useful for drawing 3d float fx, fy, cx, cy; }; } -#endif \ No newline at end of file +#endif diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index 16f1ba5..3df3593 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -31,6 +31,10 @@ // /////////////////////////////////////////////////////////////////////////////// +#include +#include +#include + #include "Visualizer.h" #include "VisualizationUtils.h" #include "RotationHelpers.h" @@ -45,12 +49,35 @@ using namespace Utilities; const int draw_shiftbits = 4; const int draw_multiplier = 1 << 4; +const std::map AUS_DESCRIPTION = { + {"AU01", "Inner Brow Raiser "}, + {"AU02", "Outer Brow Raiser "}, + {"AU04", "Brow Lowerer "}, + {"AU05", "Upper Lid Raiser "}, + {"AU06", "Cheek Raiser "}, + {"AU07", "Lid Tightener "}, + {"AU09", "Nose Wrinkler "}, + {"AU10", "Upper Lip Raiser "}, + {"AU12", "Lip Corner Puller "}, + {"AU14", "Dimpler "}, + {"AU15", "Lip Corner Depressor"}, + {"AU17", "Chin Raiser "}, + {"AU20", "Lip stretcher "}, + {"AU23", "Lip Tightener "}, + {"AU25", "Lips part "}, + {"AU26", "Jaw Drop "}, + {"AU28", "Lip Suck "}, + {"AU45", "Blink "}, + +}; + Visualizer::Visualizer(std::vector arguments) { // By default not visualizing anything this->vis_track = false; this->vis_hog = false; this->vis_align = false; + this->vis_aus = false; for (size_t i = 0; i < arguments.size(); ++i) { @@ -72,6 +99,10 @@ Visualizer::Visualizer(std::vector arguments) { vis_track = true; } + else if (arguments[i].compare("-vis-aus") == 0) + { + vis_aus = true; + } } } @@ -99,6 +130,7 @@ void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, f // Clearing other images hog_image = cv::Mat(); aligned_face_image = cv::Mat(); + action_units_image = cv::Mat(); } @@ -197,6 +229,53 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) } } +void Visualizer::SetObservationActionUnits(const std::vector >& au_intensities, + const std::vector >& au_occurences) +{ + const int NB_AUS = 17; + + const int AU_TRACKBAR_LENGTH = 400; + const int AU_TRACKBAR_HEIGHT = 10; + + const int MARGIN_X = 185; + const int MARGIN_Y = 10; + + action_units_image = cv::Mat(NB_AUS * (AU_TRACKBAR_HEIGHT + 10) + MARGIN_Y * 2, AU_TRACKBAR_LENGTH + MARGIN_X, CV_8UC3, cv::Scalar(255,255,255)); + + std::map> aus; + + // first, prepare a mapping "AU name" -> { present, intensity } + for (size_t idx = 0; idx < au_intensities.size(); idx++) { + aus[au_intensities[idx].first] = std::make_pair(au_occurences[idx].second != 0, au_intensities[idx].second); + } + + // then, build the graph + size_t idx = 0; + for (auto& au : aus) { + std::string name = au.first; + bool present = au.second.first; + double intensity = au.second.second; + + auto offset = MARGIN_Y + idx * (AU_TRACKBAR_HEIGHT + 10); + std::ostringstream au_i; + au_i << std::setprecision(2) << std::setw(4) << std::fixed << intensity; + cv::putText(action_units_image, name, cv::Point(10, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(present ? 0 : 200, 0, 0), 1, CV_AA); + cv::putText(action_units_image, AUS_DESCRIPTION.at(name), cv::Point(55, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, CV_AA); + + if(present) { + cv::putText(action_units_image, au_i.str(), cv::Point(160, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 100, 0), 1, CV_AA); + cv::rectangle(action_units_image, cv::Point(MARGIN_X, offset), + cv::Point(MARGIN_X + AU_TRACKBAR_LENGTH * intensity/5, offset + AU_TRACKBAR_HEIGHT), + cv::Scalar(128,128,128), + CV_FILLED); + } + else { + cv::putText(action_units_image, "0.00", cv::Point(160, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, CV_AA); + } + idx++; + } +} + // Eye gaze infomration drawing, first of eye landmarks then of gaze void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, double confidence) { @@ -298,6 +377,10 @@ char Visualizer::ShowObservation() { cv::imshow("hog", hog_image); } + if (vis_aus) + { + cv::imshow("action units", action_units_image); + } return cv::waitKey(1); }