r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Preparing for processing of image files, and a new image output format.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-10-30 09:43:03 +00:00
parent 8d996cce62
commit 0ac039d708
4 changed files with 67 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
gui/OpenFaceOffline/MainWindow.xaml.cs
View file

@ -335,7 +335,20 @@ namespace OpenFaceOffline
return;
}
List<List<Tuple<double, double>>> landmark_detections = ProcessImage(clnf_model, face_model_params, frame, grayFrame);
var landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayFrame, face_model_params);
// Go over all detected faces
for(int i = 0; i < landmark_detections.Count; ++i)
{
// Predict action units
var au_preds = face_analyser.PredictStaticAUs(grayFrame, landmark_detections[i]);
// Record the predictions
//String output_name = record_root +
//Recorder.RecordImg();
}
List<Point> landmark_points = new List<Point>();
@ -731,14 +744,6 @@ namespace OpenFaceOffline
}
private List<List<Tuple<double, double>>> ProcessImage(CLNF clnf_model, FaceModelParameters clnf_params, RawImage frame, RawImage grayscale_frame)
{
List<List<Tuple<double, double>>> landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayscale_frame, clnf_params);
return landmark_detections;
}
// ----------------------------------------------------------
// Mode handling (image, video)
// ----------------------------------------------------------

6
gui/OpenFaceOffline/Recorder.cs
View file

@ -296,11 +296,9 @@ namespace OpenFaceOffline
face_analyser.PostProcessOutputFile(out_filename);
}
static void RecordImg(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params,
bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG,
CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model)
static void RecordImg(string out_root, string filename, CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy)
{
// Points, pose, gaze, aus
}
}

45
lib/local/CppInerop/FaceAnalyserInterop.h
View file

@ -70,6 +70,8 @@
#undef generic
#endif
using namespace System::Collections::Generic;
#pragma managed
namespace FaceAnalyser_Interop {
@ -185,7 +187,7 @@ public:
face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file));
}
void AddNextFrame(OpenCVWrappers::RawImage^ frame, System::Collections::Generic::List<System::Tuple<double, double>^>^ landmarks, bool success, bool online, bool vis_hog) {
void AddNextFrame(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks, bool success, bool online, bool vis_hog) {
// Construct an OpenCV matric from the landmarks
cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
@ -211,7 +213,9 @@ public:
}
// Predicting AUs from a single image
System::Collections::Generic::Dictionary<System::String^, double>^ PredictStaticAUs(OpenCVWrappers::RawImage^ frame, System::Collections::Generic::List<System::Tuple<double, double>^>^ landmarks, bool success, bool vis_hog) {
System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>^
PredictStaticAUs(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks)
{
// Construct an OpenCV matric from the landmarks
cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
@ -221,26 +225,33 @@ public:
landmarks_mat.at<double>(i + landmarks->Count, 0) = landmarks[i]->Item2;
}
face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, false, vis_hog);
auto aus = face_analyser->PredictStaticAUs(frame->Mat, landmarks_mat, false);
face_analyser->GetLatestHOG(*hog_features, *num_rows, *num_cols);
auto AU_predictions_intensity = aus.first;
auto AU_predictions_occurence = aus.second;
face_analyser->GetLatestAlignedFace(*aligned_face);
auto au_intensities = gcnew Dictionary<System::String^, double>();
auto au_occurences = gcnew Dictionary<System::String^, double>();
*good_frame = success;
if (vis_hog)
for (auto p : AU_predictions_intensity)
{
*visualisation = face_analyser->GetLatestHOGDescriptorVisualisation();
au_intensities->Add(gcnew System::String(p.first.c_str()), p.second);
}
for (auto p : AU_predictions_occurence)
{
au_occurences->Add(gcnew System::String(p.first.c_str()), p.second);
}
return gcnew System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>(au_intensities, au_occurences);
}
System::Collections::Generic::List<System::String^>^ GetClassActionUnitsNames()
List<System::String^>^ GetClassActionUnitsNames()
{
auto names = face_analyser->GetAUClassNames();
auto names_ret = gcnew System::Collections::Generic::List<System::String^>();
auto names_ret = gcnew List<System::String^>();
for(std::string name : names)
{
@ -251,11 +262,11 @@ public:
}
System::Collections::Generic::List<System::String^>^ GetRegActionUnitsNames()
List<System::String^>^ GetRegActionUnitsNames()
{
auto names = face_analyser->GetAURegNames();
auto names_ret = gcnew System::Collections::Generic::List<System::String^>();
auto names_ret = gcnew List<System::String^>();
for(std::string name : names)
{
@ -266,10 +277,10 @@ public:
}
System::Collections::Generic::Dictionary<System::String^, double>^ GetCurrentAUsClass()
Dictionary<System::String^, double>^ GetCurrentAUsClass()
{
auto classes = face_analyser->GetCurrentAUsClass();
auto au_classes = gcnew System::Collections::Generic::Dictionary<System::String^, double>();
auto au_classes = gcnew Dictionary<System::String^, double>();
for(auto p: classes)
{
@ -278,10 +289,10 @@ public:
return au_classes;
}
System::Collections::Generic::Dictionary<System::String^, double>^ GetCurrentAUsReg()
Dictionary<System::String^, double>^ GetCurrentAUsReg()
{
auto preds = face_analyser->GetCurrentAUsReg();
auto au_preds = gcnew System::Collections::Generic::Dictionary<System::String^, double>();
auto au_preds = gcnew Dictionary<System::String^, double>();
for(auto p: preds)
{

44
lib/local/CppInerop/LandmarkDetectorInterop.h
View file

@ -68,6 +68,8 @@
#undef generic
#endif
using namespace System::Collections::Generic;
#pragma managed
namespace CppInterop {
@ -202,9 +204,9 @@ namespace CppInterop {
return ::LandmarkDetector::DetectLandmarksInImage(image->Mat, *clnf, *modelParams->getParams());
}
System::Collections::Generic::List<System::Collections::Generic::List<System::Tuple<double,double>^>^>^ DetectMultiFaceLandmarksInImage(OpenCVWrappers::RawImage^ image, FaceModelParameters^ modelParams) {
List<List<System::Tuple<double,double>^>^>^ DetectMultiFaceLandmarksInImage(OpenCVWrappers::RawImage^ image, FaceModelParameters^ modelParams) {
auto all_landmarks = gcnew System::Collections::Generic::List<System::Collections::Generic::List<System::Tuple<double,double>^>^>();
auto all_landmarks = gcnew List<List<System::Tuple<double,double>^>^>();
// Detect faces in an image
vector<cv::Rect_<double> > face_detections;
@ -224,7 +226,7 @@ namespace CppInterop {
// if there are multiple detections go through them
bool success = ::LandmarkDetector::DetectLandmarksInImage(image->Mat, face_detections[face], *clnf, *modelParams->getParams());
auto landmarks_curr = gcnew System::Collections::Generic::List<System::Tuple<double,double>^>();
auto landmarks_curr = gcnew List<System::Tuple<double,double>^>();
if(clnf->detected_landmarks.cols == 1)
{
int n = clnf->detected_landmarks.rows / 2;
@ -248,7 +250,7 @@ namespace CppInterop {
return all_landmarks;
}
void GetPoseWRTCamera(System::Collections::Generic::List<double>^ pose, double fx, double fy, double cx, double cy) {
void GetPoseWRTCamera(List<double>^ pose, double fx, double fy, double cx, double cy) {
auto pose_vec = ::LandmarkDetector::GetPoseWRTCamera(*clnf, fx, fy, cx, cy);
pose->Clear();
for(int i = 0; i < 6; ++i)
@ -257,7 +259,7 @@ namespace CppInterop {
}
}
void GetPose(System::Collections::Generic::List<double>^ pose, double fx, double fy, double cx, double cy) {
void GetPose(List<double>^ pose, double fx, double fy, double cx, double cy) {
auto pose_vec = ::LandmarkDetector::GetPose(*clnf, fx, fy, cx, cy);
pose->Clear();
for(int i = 0; i < 6; ++i)
@ -266,7 +268,7 @@ namespace CppInterop {
}
}
System::Collections::Generic::List<System::Tuple<double,double>^>^ CalculateVisibleLandmarks() {
List<System::Tuple<double,double>^>^ CalculateVisibleLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateVisibleLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double,double>^>();
@ -277,10 +279,10 @@ namespace CppInterop {
return landmarks;
}
System::Collections::Generic::List<System::Tuple<double, double>^>^ CalculateAllLandmarks() {
List<System::Tuple<double, double>^>^ CalculateAllLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateAllLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>();
auto landmarks = gcnew List<System::Tuple<double, double>^>();
for (cv::Point2d p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<double, double>(p.x, p.y));
}
@ -288,7 +290,7 @@ namespace CppInterop {
return landmarks;
}
System::Collections::Generic::List<System::Tuple<double, double>^>^ CalculateAllEyeLandmarks() {
List<System::Tuple<double, double>^>^ CalculateAllEyeLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateAllEyeLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>();
@ -299,7 +301,7 @@ namespace CppInterop {
return landmarks;
}
System::Collections::Generic::List<System::Tuple<double, double>^>^ CalculateVisibleEyeLandmarks() {
List<System::Tuple<double, double>^>^ CalculateVisibleEyeLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateVisibleEyeLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>();
@ -310,11 +312,11 @@ namespace CppInterop {
return landmarks;
}
System::Collections::Generic::List<System::Windows::Media::Media3D::Point3D>^ Calculate3DLandmarks(double fx, double fy, double cx, double cy) {
List<System::Windows::Media::Media3D::Point3D>^ Calculate3DLandmarks(double fx, double fy, double cx, double cy) {
cv::Mat_<double> shape3D = clnf->GetShape(fx, fy, cx, cy);
auto landmarks_3D = gcnew System::Collections::Generic::List<System::Windows::Media::Media3D::Point3D>();
auto landmarks_3D = gcnew List<System::Windows::Media::Media3D::Point3D>();
for(int i = 0; i < shape3D.cols; ++i)
{
@ -326,7 +328,7 @@ namespace CppInterop {
// Static functions from the LandmarkDetector namespace.
void DrawLandmarks(OpenCVWrappers::RawImage^ img, System::Collections::Generic::List<System::Windows::Point>^ landmarks) {
void DrawLandmarks(OpenCVWrappers::RawImage^ img, List<System::Windows::Point>^ landmarks) {
vector<cv::Point> vecLandmarks;
@ -339,13 +341,13 @@ namespace CppInterop {
}
System::Collections::Generic::List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateBox(float fx, float fy, float cx, float cy) {
List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateBox(float fx, float fy, float cx, float cy) {
cv::Vec6d pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy);
vector<pair<cv::Point2d, cv::Point2d>> vecLines = ::LandmarkDetector::CalculateBox(pose, fx, fy, cx, cy);
auto lines = gcnew System::Collections::Generic::List<System::Tuple<System::Windows::Point,System::Windows::Point>^>();
auto lines = gcnew List<System::Tuple<System::Windows::Point,System::Windows::Point>^>();
for(pair<cv::Point2d, cv::Point2d> line : vecLines) {
lines->Add(gcnew System::Tuple<System::Windows::Point, System::Windows::Point>(System::Windows::Point(line.first.x, line.first.y), System::Windows::Point(line.second.x, line.second.y)));
@ -354,7 +356,7 @@ namespace CppInterop {
return lines;
}
void DrawBox(System::Collections::Generic::List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) {
void DrawBox(List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) {
cv::Scalar color = cv::Scalar(r,g,b,1);
vector<pair<cv::Point, cv::Point>> vecLines;
@ -378,9 +380,9 @@ namespace CppInterop {
}
// Getting the non-rigid shape parameters describing the facial expression
System::Collections::Generic::List<double>^ GetNonRigidParams()
List<double>^ GetNonRigidParams()
{
auto non_rigid_params = gcnew System::Collections::Generic::List<double>();
auto non_rigid_params = gcnew List<double>();
for (int i = 0; i < clnf->params_local.rows; ++i)
{
@ -391,9 +393,9 @@ namespace CppInterop {
}
// Getting the rigid shape parameters describing face scale rotation and translation (scale,rotx,roty,rotz,tx,ty)
System::Collections::Generic::List<double>^ GetRigidParams()
List<double>^ GetRigidParams()
{
auto rigid_params = gcnew System::Collections::Generic::List<double>();
auto rigid_params = gcnew List<double>();
for (size_t i = 0; i < 6; ++i)
{
@ -403,7 +405,7 @@ namespace CppInterop {
}
// Rigid params followed by non-rigid ones
System::Collections::Generic::List<double>^ GetParams()
List<double>^ GetParams()
{
auto all_params = GetRigidParams();
all_params->AddRange(GetNonRigidParams());