sustaining_gazes/lib/local/CppInerop/FaceAnalyserInterop.h

294 lines
8.2 KiB
C
Raw Normal View History

2016-05-20 22:48:43 +02:00
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
2016-05-20 22:48:43 +02:00
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
2016-05-20 22:48:43 +02:00
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
2016-05-20 22:48:43 +02:00
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace: an open source facial behavior analysis toolkit
// Tadas Baltru<72>aitis, Peter Robinson, and Louis-Philippe Morency
// in IEEE Winter Conference on Applications of Computer Vision, 2016
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltru<72>aitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
// Tadas Baltru<72>aitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
// Constrained Local Neural Fields for robust facial landmark detection in the wild.
// Tadas Baltru<72>aitis, Peter Robinson, and Louis-Philippe Morency.
// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.
//
///////////////////////////////////////////////////////////////////////////////
// FaceAnalyser_Interop.h
#ifndef __FACE_ANALYSER_INTEROP_h_
#define __FACE_ANALYSER_INTEROP_h_
#pragma once
// Include all the unmanaged things we need.
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
#pragma unmanaged
// Allows to overcome boost name clash stuff with C++ CLI
#ifdef __cplusplus_cli
#define generic __identifier(generic)
#endif
2016-05-20 22:48:43 +02:00
#include <opencv2/core/core.hpp>
#include "opencv2/objdetect.hpp"
#include "opencv2/calib3d.hpp"
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <OpenCVWrappers.h>
#include <Face_utils.h>
#include <FaceAnalyser.h>
#include <VisualizationUtils.h>
2016-05-20 22:48:43 +02:00
// Boost stuff
#include <filesystem.hpp>
#include <filesystem/fstream.hpp>
#ifdef __cplusplus_cli
#undef generic
#endif
using namespace System::Collections::Generic;
2016-05-20 22:48:43 +02:00
#pragma managed
namespace FaceAnalyser_Interop {
public ref class FaceAnalyserManaged
{
private:
FaceAnalysis::FaceAnalyser* face_analyser;
// The actual descriptors (for visualisation and output)
cv::Mat_<float>* hog_features;
2016-05-20 22:48:43 +02:00
cv::Mat* aligned_face;
int* num_rows;
int* num_cols;
public:
FaceAnalyserManaged(System::String^ root, bool dynamic, int output_width, bool mask_aligned)
2016-05-20 22:48:43 +02:00
{
string root_std = msclr::interop::marshal_as<std::string>(root);
FaceAnalysis::FaceAnalyserParameters params(root_std);
if (!dynamic)
{
params.OptimizeForImages();
}
params.setAlignedOutput(output_width, -1.0, mask_aligned);
face_analyser = new FaceAnalysis::FaceAnalyser(params);
2016-05-20 22:48:43 +02:00
hog_features = new cv::Mat_<float>();
2016-05-20 22:48:43 +02:00
aligned_face = new cv::Mat();
num_rows = new int;
num_cols = new int;
}
void PostProcessOutputFile(System::String^ file)
2016-12-09 16:06:04 +01:00
{
face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file));
2016-12-09 16:06:04 +01:00
}
2018-01-20 10:09:59 +01:00
void AddNextFrame(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks, bool success, bool online) {
2016-05-20 22:48:43 +02:00
// Construct an OpenCV matric from the landmarks
cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
for (int i = 0; i < landmarks->Count; ++i)
{
landmarks_mat.at<double>(i, 0) = landmarks[i]->Item1;
landmarks_mat.at<double>(i + landmarks->Count, 0) = landmarks[i]->Item2;
}
//(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam());
face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, online);
2016-05-20 22:48:43 +02:00
cv::Mat_<double> hog_d;
face_analyser->GetLatestHOG(hog_d, *num_rows, *num_cols);
hog_d.convertTo(*hog_features, CV_64F);
2016-05-20 22:48:43 +02:00
face_analyser->GetLatestAlignedFace(*aligned_face);
2018-01-24 09:11:44 +01:00
2016-05-20 22:48:43 +02:00
}
2017-10-30 08:50:33 +01:00
// Predicting AUs from a single image
System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>^
2018-01-20 10:09:59 +01:00
PredictStaticAUsAndComputeFeatures(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks)
{
2017-10-30 08:50:33 +01:00
// Construct an OpenCV matric from the landmarks
cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
for (int i = 0; i < landmarks->Count; ++i)
{
landmarks_mat.at<double>(i, 0) = landmarks[i]->Item1;
landmarks_mat.at<double>(i + landmarks->Count, 0) = landmarks[i]->Item2;
}
face_analyser->PredictStaticAUsAndComputeFeatures(frame->Mat, landmarks_mat);
2017-10-30 08:50:33 +01:00
2018-01-10 18:43:05 +01:00
// Set the computed appearance features
cv::Mat_<double> hog_tmp;
face_analyser->GetLatestHOG(hog_tmp, *num_rows, *num_cols);
hog_tmp.convertTo(*hog_features, CV_32F);
2018-01-10 18:43:05 +01:00
face_analyser->GetLatestAlignedFace(*aligned_face);
// Set the computed AUs
auto AU_predictions_intensity = face_analyser->GetCurrentAUsReg();
auto AU_predictions_occurence = face_analyser->GetCurrentAUsClass();
2017-10-30 08:50:33 +01:00
auto au_intensities = gcnew Dictionary<System::String^, double>();
auto au_occurences = gcnew Dictionary<System::String^, double>();
2017-10-30 08:50:33 +01:00
for (auto p : AU_predictions_intensity)
{
au_intensities->Add(gcnew System::String(p.first.c_str()), p.second);
}
2017-10-30 08:50:33 +01:00
for (auto p : AU_predictions_occurence)
2017-10-30 08:50:33 +01:00
{
au_occurences->Add(gcnew System::String(p.first.c_str()), p.second);
2017-10-30 08:50:33 +01:00
}
return gcnew System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>(au_intensities, au_occurences);
2017-10-30 08:50:33 +01:00
}
List<System::String^>^ GetClassActionUnitsNames()
2016-05-20 22:48:43 +02:00
{
auto names = face_analyser->GetAUClassNames();
auto names_ret = gcnew List<System::String^>();
2016-05-20 22:48:43 +02:00
for(std::string name : names)
{
names_ret->Add(gcnew System::String(name.c_str()));
}
return names_ret;
}
List<System::String^>^ GetRegActionUnitsNames()
2016-05-20 22:48:43 +02:00
{
auto names = face_analyser->GetAURegNames();
auto names_ret = gcnew List<System::String^>();
2016-05-20 22:48:43 +02:00
for(std::string name : names)
{
names_ret->Add(gcnew System::String(name.c_str()));
}
return names_ret;
}
Dictionary<System::String^, double>^ GetCurrentAUsClass()
2016-05-20 22:48:43 +02:00
{
auto classes = face_analyser->GetCurrentAUsClass();
auto au_classes = gcnew Dictionary<System::String^, double>();
2016-05-20 22:48:43 +02:00
for(auto p: classes)
{
au_classes->Add(gcnew System::String(p.first.c_str()), p.second);
}
return au_classes;
}
Dictionary<System::String^, double>^ GetCurrentAUsReg()
2016-05-20 22:48:43 +02:00
{
auto preds = face_analyser->GetCurrentAUsReg();
auto au_preds = gcnew Dictionary<System::String^, double>();
2016-05-20 22:48:43 +02:00
for(auto p: preds)
{
au_preds->Add(gcnew System::String(p.first.c_str()), p.second);
}
return au_preds;
}
OpenCVWrappers::RawImage^ GetLatestAlignedFace() {
OpenCVWrappers::RawImage^ face_aligned_image = gcnew OpenCVWrappers::RawImage(*aligned_face);
return face_aligned_image;
}
OpenCVWrappers::RawImage^ GetLatestHOGFeature() {
OpenCVWrappers::RawImage^ HOG_feature = gcnew OpenCVWrappers::RawImage(*hog_features);
return HOG_feature;
}
2016-05-20 22:48:43 +02:00
// As the number of HOG rows and columns might not be known in advance, have methods for querying them
int GetHOGRows()
{
return *num_rows;
}
int GetHOGCols()
{
return *num_cols;
}
// The number of channels is always the same
int GetHOGChannels()
{
return 31;
}
2016-05-20 22:48:43 +02:00
void Reset()
{
face_analyser->Reset();
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!FaceAnalyserManaged()
{
delete hog_features;
delete aligned_face;
delete num_cols;
delete num_rows;
delete face_analyser;
}
// Destructor. Called on explicit Dispose() only.
~FaceAnalyserManaged()
{
this->!FaceAnalyserManaged();
}
};
}
#endif