Added support for AUs visualization to FeatureExtractor
This commit is contained in:
parent
0a769e4424
commit
dba96e1551
3 changed files with 90 additions and 2 deletions
|
@ -205,6 +205,7 @@ int main (int argc, char **argv)
|
|||
visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
|
||||
visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
|
||||
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
|
||||
visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
|
||||
visualizer.SetFps(fps_tracker.GetFPS());
|
||||
|
||||
// detect key presses
|
||||
|
|
|
@ -66,7 +66,9 @@ namespace Utilities
|
|||
|
||||
// Pose related observations
|
||||
void SetObservationPose(const cv::Vec6d& pose, double confidence);
|
||||
|
||||
|
||||
void SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
|
||||
|
||||
// Gaze related observations
|
||||
void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2d>& eye_landmarks, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence);
|
||||
|
||||
|
@ -88,6 +90,7 @@ namespace Utilities
|
|||
bool vis_track;
|
||||
bool vis_hog;
|
||||
bool vis_align;
|
||||
bool vis_aus;
|
||||
|
||||
// Can be adjusted to show less confident frames
|
||||
double visualisation_boundary = 0.4;
|
||||
|
@ -99,10 +102,11 @@ namespace Utilities
|
|||
cv::Mat tracked_image;
|
||||
cv::Mat hog_image;
|
||||
cv::Mat aligned_face_image;
|
||||
cv::Mat action_units_image;
|
||||
|
||||
// Useful for drawing 3d
|
||||
float fx, fy, cx, cy;
|
||||
|
||||
};
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
@ -31,6 +31,10 @@
|
|||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#include <map>
|
||||
|
||||
#include "Visualizer.h"
|
||||
#include "VisualizationUtils.h"
|
||||
#include "RotationHelpers.h"
|
||||
|
@ -45,12 +49,35 @@ using namespace Utilities;
|
|||
const int draw_shiftbits = 4;
|
||||
const int draw_multiplier = 1 << 4;
|
||||
|
||||
const std::map<std::string, std::string> AUS_DESCRIPTION = {
|
||||
{"AU01", "Inner Brow Raiser "},
|
||||
{"AU02", "Outer Brow Raiser "},
|
||||
{"AU04", "Brow Lowerer "},
|
||||
{"AU05", "Upper Lid Raiser "},
|
||||
{"AU06", "Cheek Raiser "},
|
||||
{"AU07", "Lid Tightener "},
|
||||
{"AU09", "Nose Wrinkler "},
|
||||
{"AU10", "Upper Lip Raiser "},
|
||||
{"AU12", "Lip Corner Puller "},
|
||||
{"AU14", "Dimpler "},
|
||||
{"AU15", "Lip Corner Depressor"},
|
||||
{"AU17", "Chin Raiser "},
|
||||
{"AU20", "Lip stretcher "},
|
||||
{"AU23", "Lip Tightener "},
|
||||
{"AU25", "Lips part "},
|
||||
{"AU26", "Jaw Drop "},
|
||||
{"AU28", "Lip Suck "},
|
||||
{"AU45", "Blink "},
|
||||
|
||||
};
|
||||
|
||||
Visualizer::Visualizer(std::vector<std::string> arguments)
|
||||
{
|
||||
// By default not visualizing anything
|
||||
this->vis_track = false;
|
||||
this->vis_hog = false;
|
||||
this->vis_align = false;
|
||||
this->vis_aus = false;
|
||||
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
|
@ -72,6 +99,10 @@ Visualizer::Visualizer(std::vector<std::string> arguments)
|
|||
{
|
||||
vis_track = true;
|
||||
}
|
||||
else if (arguments[i].compare("-vis-aus") == 0)
|
||||
{
|
||||
vis_aus = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -99,6 +130,7 @@ void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, f
|
|||
// Clearing other images
|
||||
hog_image = cv::Mat();
|
||||
aligned_face_image = cv::Mat();
|
||||
action_units_image = cv::Mat();
|
||||
|
||||
}
|
||||
|
||||
|
@ -197,6 +229,53 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
|
|||
}
|
||||
}
|
||||
|
||||
void Visualizer::SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities,
|
||||
const std::vector<std::pair<std::string, double> >& au_occurences)
|
||||
{
|
||||
const int NB_AUS = 17;
|
||||
|
||||
const int AU_TRACKBAR_LENGTH = 400;
|
||||
const int AU_TRACKBAR_HEIGHT = 10;
|
||||
|
||||
const int MARGIN_X = 185;
|
||||
const int MARGIN_Y = 10;
|
||||
|
||||
action_units_image = cv::Mat(NB_AUS * (AU_TRACKBAR_HEIGHT + 10) + MARGIN_Y * 2, AU_TRACKBAR_LENGTH + MARGIN_X, CV_8UC3, cv::Scalar(255,255,255));
|
||||
|
||||
std::map<std::string, std::pair<bool, double>> aus;
|
||||
|
||||
// first, prepare a mapping "AU name" -> { present, intensity }
|
||||
for (size_t idx = 0; idx < au_intensities.size(); idx++) {
|
||||
aus[au_intensities[idx].first] = std::make_pair(au_occurences[idx].second != 0, au_intensities[idx].second);
|
||||
}
|
||||
|
||||
// then, build the graph
|
||||
size_t idx = 0;
|
||||
for (auto& au : aus) {
|
||||
std::string name = au.first;
|
||||
bool present = au.second.first;
|
||||
double intensity = au.second.second;
|
||||
|
||||
auto offset = MARGIN_Y + idx * (AU_TRACKBAR_HEIGHT + 10);
|
||||
std::ostringstream au_i;
|
||||
au_i << std::setprecision(2) << std::setw(4) << std::fixed << intensity;
|
||||
cv::putText(action_units_image, name, cv::Point(10, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(present ? 0 : 200, 0, 0), 1, CV_AA);
|
||||
cv::putText(action_units_image, AUS_DESCRIPTION.at(name), cv::Point(55, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, CV_AA);
|
||||
|
||||
if(present) {
|
||||
cv::putText(action_units_image, au_i.str(), cv::Point(160, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 100, 0), 1, CV_AA);
|
||||
cv::rectangle(action_units_image, cv::Point(MARGIN_X, offset),
|
||||
cv::Point(MARGIN_X + AU_TRACKBAR_LENGTH * intensity/5, offset + AU_TRACKBAR_HEIGHT),
|
||||
cv::Scalar(128,128,128),
|
||||
CV_FILLED);
|
||||
}
|
||||
else {
|
||||
cv::putText(action_units_image, "0.00", cv::Point(160, offset + 10), CV_FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, CV_AA);
|
||||
}
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
|
||||
// Eye gaze infomration drawing, first of eye landmarks then of gaze
|
||||
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence)
|
||||
{
|
||||
|
@ -298,6 +377,10 @@ char Visualizer::ShowObservation()
|
|||
{
|
||||
cv::imshow("hog", hog_image);
|
||||
}
|
||||
if (vis_aus)
|
||||
{
|
||||
cv::imshow("action units", action_units_image);
|
||||
}
|
||||
return cv::waitKey(1);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue