r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Preparing for processing of image files, and a new image output format.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-10-30 09:43:03 +00:00
parent 8d996cce62
commit 0ac039d708
4 changed files with 67 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
gui/OpenFaceOffline/MainWindow.xaml.cs
View file

@ -335,7 +335,20 @@ namespace OpenFaceOffline
return; return;
} }
List<List<Tuple<double, double>>> landmark_detections = ProcessImage(clnf_model, face_model_params, frame, grayFrame); var landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayFrame, face_model_params);
// Go over all detected faces
for(int i = 0; i < landmark_detections.Count; ++i)
{
// Predict action units
var au_preds = face_analyser.PredictStaticAUs(grayFrame, landmark_detections[i]);
// Record the predictions
//String output_name = record_root +
//Recorder.RecordImg();
}
List<Point> landmark_points = new List<Point>(); List<Point> landmark_points = new List<Point>();
@ -731,14 +744,6 @@ namespace OpenFaceOffline
} }
private List<List<Tuple<double, double>>> ProcessImage(CLNF clnf_model, FaceModelParameters clnf_params, RawImage frame, RawImage grayscale_frame)
{
List<List<Tuple<double, double>>> landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayscale_frame, clnf_params);
return landmark_detections;
}
// ---------------------------------------------------------- // ----------------------------------------------------------
// Mode handling (image, video) // Mode handling (image, video)
// ---------------------------------------------------------- // ----------------------------------------------------------

6
gui/OpenFaceOffline/Recorder.cs
View file

@ -296,11 +296,9 @@ namespace OpenFaceOffline
face_analyser.PostProcessOutputFile(out_filename); face_analyser.PostProcessOutputFile(out_filename);
} }
static void RecordImg(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, static void RecordImg(string out_root, string filename, CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy)
bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG,
CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model)
{ {
// Points, pose, gaze, aus
} }
} }

45
lib/local/CppInerop/FaceAnalyserInterop.h
View file

@ -70,6 +70,8 @@
#undef generic #undef generic
#endif #endif
using namespace System::Collections::Generic;
#pragma managed #pragma managed
namespace FaceAnalyser_Interop { namespace FaceAnalyser_Interop {
@ -185,7 +187,7 @@ public:
face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file)); face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file));
} }
void AddNextFrame(OpenCVWrappers::RawImage^ frame, System::Collections::Generic::List<System::Tuple<double, double>^>^ landmarks, bool success, bool online, bool vis_hog) { void AddNextFrame(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks, bool success, bool online, bool vis_hog) {
// Construct an OpenCV matric from the landmarks // Construct an OpenCV matric from the landmarks
cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0); cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
@ -211,7 +213,9 @@ public:
} }
// Predicting AUs from a single image // Predicting AUs from a single image
System::Collections::Generic::Dictionary<System::String^, double>^ PredictStaticAUs(OpenCVWrappers::RawImage^ frame, System::Collections::Generic::List<System::Tuple<double, double>^>^ landmarks, bool success, bool vis_hog) { System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>^
PredictStaticAUs(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks)
{
// Construct an OpenCV matric from the landmarks // Construct an OpenCV matric from the landmarks
cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0); cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
@ -221,26 +225,33 @@ public:
landmarks_mat.at<double>(i + landmarks->Count, 0) = landmarks[i]->Item2; landmarks_mat.at<double>(i + landmarks->Count, 0) = landmarks[i]->Item2;
} }
face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, false, vis_hog); auto aus = face_analyser->PredictStaticAUs(frame->Mat, landmarks_mat, false);
face_analyser->GetLatestHOG(*hog_features, *num_rows, *num_cols); auto AU_predictions_intensity = aus.first;
auto AU_predictions_occurence = aus.second;
face_analyser->GetLatestAlignedFace(*aligned_face); auto au_intensities = gcnew Dictionary<System::String^, double>();
auto au_occurences = gcnew Dictionary<System::String^, double>();
*good_frame = success; for (auto p : AU_predictions_intensity)
if (vis_hog)
{ {
*visualisation = face_analyser->GetLatestHOGDescriptorVisualisation(); au_intensities->Add(gcnew System::String(p.first.c_str()), p.second);
} }
for (auto p : AU_predictions_occurence)
{
au_occurences->Add(gcnew System::String(p.first.c_str()), p.second);
}
return gcnew System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>(au_intensities, au_occurences);
} }
System::Collections::Generic::List<System::String^>^ GetClassActionUnitsNames() List<System::String^>^ GetClassActionUnitsNames()
{ {
auto names = face_analyser->GetAUClassNames(); auto names = face_analyser->GetAUClassNames();
auto names_ret = gcnew System::Collections::Generic::List<System::String^>(); auto names_ret = gcnew List<System::String^>();
for(std::string name : names) for(std::string name : names)
{ {
@ -251,11 +262,11 @@ public:
} }
System::Collections::Generic::List<System::String^>^ GetRegActionUnitsNames() List<System::String^>^ GetRegActionUnitsNames()
{ {
auto names = face_analyser->GetAURegNames(); auto names = face_analyser->GetAURegNames();
auto names_ret = gcnew System::Collections::Generic::List<System::String^>(); auto names_ret = gcnew List<System::String^>();
for(std::string name : names) for(std::string name : names)
{ {
@ -266,10 +277,10 @@ public:
} }
System::Collections::Generic::Dictionary<System::String^, double>^ GetCurrentAUsClass() Dictionary<System::String^, double>^ GetCurrentAUsClass()
{ {
auto classes = face_analyser->GetCurrentAUsClass(); auto classes = face_analyser->GetCurrentAUsClass();
auto au_classes = gcnew System::Collections::Generic::Dictionary<System::String^, double>(); auto au_classes = gcnew Dictionary<System::String^, double>();
for(auto p: classes) for(auto p: classes)
{ {
@ -278,10 +289,10 @@ public:
return au_classes; return au_classes;
} }
System::Collections::Generic::Dictionary<System::String^, double>^ GetCurrentAUsReg() Dictionary<System::String^, double>^ GetCurrentAUsReg()
{ {
auto preds = face_analyser->GetCurrentAUsReg(); auto preds = face_analyser->GetCurrentAUsReg();
auto au_preds = gcnew System::Collections::Generic::Dictionary<System::String^, double>(); auto au_preds = gcnew Dictionary<System::String^, double>();
for(auto p: preds) for(auto p: preds)
{ {

44
lib/local/CppInerop/LandmarkDetectorInterop.h
View file

@ -68,6 +68,8 @@
#undef generic #undef generic
#endif #endif
using namespace System::Collections::Generic;
#pragma managed #pragma managed
namespace CppInterop { namespace CppInterop {
@ -202,9 +204,9 @@ namespace CppInterop {
return ::LandmarkDetector::DetectLandmarksInImage(image->Mat, *clnf, *modelParams->getParams()); return ::LandmarkDetector::DetectLandmarksInImage(image->Mat, *clnf, *modelParams->getParams());
} }
System::Collections::Generic::List<System::Collections::Generic::List<System::Tuple<double,double>^>^>^ DetectMultiFaceLandmarksInImage(OpenCVWrappers::RawImage^ image, FaceModelParameters^ modelParams) { List<List<System::Tuple<double,double>^>^>^ DetectMultiFaceLandmarksInImage(OpenCVWrappers::RawImage^ image, FaceModelParameters^ modelParams) {
auto all_landmarks = gcnew System::Collections::Generic::List<System::Collections::Generic::List<System::Tuple<double,double>^>^>(); auto all_landmarks = gcnew List<List<System::Tuple<double,double>^>^>();
// Detect faces in an image // Detect faces in an image
vector<cv::Rect_<double> > face_detections; vector<cv::Rect_<double> > face_detections;
@ -224,7 +226,7 @@ namespace CppInterop {
// if there are multiple detections go through them // if there are multiple detections go through them
bool success = ::LandmarkDetector::DetectLandmarksInImage(image->Mat, face_detections[face], *clnf, *modelParams->getParams()); bool success = ::LandmarkDetector::DetectLandmarksInImage(image->Mat, face_detections[face], *clnf, *modelParams->getParams());
auto landmarks_curr = gcnew System::Collections::Generic::List<System::Tuple<double,double>^>(); auto landmarks_curr = gcnew List<System::Tuple<double,double>^>();
if(clnf->detected_landmarks.cols == 1) if(clnf->detected_landmarks.cols == 1)
{ {
int n = clnf->detected_landmarks.rows / 2; int n = clnf->detected_landmarks.rows / 2;
@ -248,7 +250,7 @@ namespace CppInterop {
return all_landmarks; return all_landmarks;
} }
void GetPoseWRTCamera(System::Collections::Generic::List<double>^ pose, double fx, double fy, double cx, double cy) { void GetPoseWRTCamera(List<double>^ pose, double fx, double fy, double cx, double cy) {
auto pose_vec = ::LandmarkDetector::GetPoseWRTCamera(*clnf, fx, fy, cx, cy); auto pose_vec = ::LandmarkDetector::GetPoseWRTCamera(*clnf, fx, fy, cx, cy);
pose->Clear(); pose->Clear();
for(int i = 0; i < 6; ++i) for(int i = 0; i < 6; ++i)
@ -257,7 +259,7 @@ namespace CppInterop {
} }
} }
void GetPose(System::Collections::Generic::List<double>^ pose, double fx, double fy, double cx, double cy) { void GetPose(List<double>^ pose, double fx, double fy, double cx, double cy) {
auto pose_vec = ::LandmarkDetector::GetPose(*clnf, fx, fy, cx, cy); auto pose_vec = ::LandmarkDetector::GetPose(*clnf, fx, fy, cx, cy);
pose->Clear(); pose->Clear();
for(int i = 0; i < 6; ++i) for(int i = 0; i < 6; ++i)
@ -266,7 +268,7 @@ namespace CppInterop {
} }
} }
System::Collections::Generic::List<System::Tuple<double,double>^>^ CalculateVisibleLandmarks() { List<System::Tuple<double,double>^>^ CalculateVisibleLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateVisibleLandmarks(*clnf); vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateVisibleLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double,double>^>(); auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double,double>^>();
@ -277,10 +279,10 @@ namespace CppInterop {
return landmarks; return landmarks;
} }
System::Collections::Generic::List<System::Tuple<double, double>^>^ CalculateAllLandmarks() { List<System::Tuple<double, double>^>^ CalculateAllLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateAllLandmarks(*clnf); vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateAllLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>(); auto landmarks = gcnew List<System::Tuple<double, double>^>();
for (cv::Point2d p : vecLandmarks) { for (cv::Point2d p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<double, double>(p.x, p.y)); landmarks->Add(gcnew System::Tuple<double, double>(p.x, p.y));
} }
@ -288,7 +290,7 @@ namespace CppInterop {
return landmarks; return landmarks;
} }
System::Collections::Generic::List<System::Tuple<double, double>^>^ CalculateAllEyeLandmarks() { List<System::Tuple<double, double>^>^ CalculateAllEyeLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateAllEyeLandmarks(*clnf); vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateAllEyeLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>(); auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>();
@ -299,7 +301,7 @@ namespace CppInterop {
return landmarks; return landmarks;
} }
System::Collections::Generic::List<System::Tuple<double, double>^>^ CalculateVisibleEyeLandmarks() { List<System::Tuple<double, double>^>^ CalculateVisibleEyeLandmarks() {
vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateVisibleEyeLandmarks(*clnf); vector<cv::Point2d> vecLandmarks = ::LandmarkDetector::CalculateVisibleEyeLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>(); auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<double, double>^>();
@ -310,11 +312,11 @@ namespace CppInterop {
return landmarks; return landmarks;
} }
System::Collections::Generic::List<System::Windows::Media::Media3D::Point3D>^ Calculate3DLandmarks(double fx, double fy, double cx, double cy) { List<System::Windows::Media::Media3D::Point3D>^ Calculate3DLandmarks(double fx, double fy, double cx, double cy) {
cv::Mat_<double> shape3D = clnf->GetShape(fx, fy, cx, cy); cv::Mat_<double> shape3D = clnf->GetShape(fx, fy, cx, cy);
auto landmarks_3D = gcnew System::Collections::Generic::List<System::Windows::Media::Media3D::Point3D>(); auto landmarks_3D = gcnew List<System::Windows::Media::Media3D::Point3D>();
for(int i = 0; i < shape3D.cols; ++i) for(int i = 0; i < shape3D.cols; ++i)
{ {
@ -326,7 +328,7 @@ namespace CppInterop {
// Static functions from the LandmarkDetector namespace. // Static functions from the LandmarkDetector namespace.
void DrawLandmarks(OpenCVWrappers::RawImage^ img, System::Collections::Generic::List<System::Windows::Point>^ landmarks) { void DrawLandmarks(OpenCVWrappers::RawImage^ img, List<System::Windows::Point>^ landmarks) {
vector<cv::Point> vecLandmarks; vector<cv::Point> vecLandmarks;
@ -339,13 +341,13 @@ namespace CppInterop {
} }
System::Collections::Generic::List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateBox(float fx, float fy, float cx, float cy) { List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateBox(float fx, float fy, float cx, float cy) {
cv::Vec6d pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy); cv::Vec6d pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy);
vector<pair<cv::Point2d, cv::Point2d>> vecLines = ::LandmarkDetector::CalculateBox(pose, fx, fy, cx, cy); vector<pair<cv::Point2d, cv::Point2d>> vecLines = ::LandmarkDetector::CalculateBox(pose, fx, fy, cx, cy);
auto lines = gcnew System::Collections::Generic::List<System::Tuple<System::Windows::Point,System::Windows::Point>^>(); auto lines = gcnew List<System::Tuple<System::Windows::Point,System::Windows::Point>^>();
for(pair<cv::Point2d, cv::Point2d> line : vecLines) { for(pair<cv::Point2d, cv::Point2d> line : vecLines) {
lines->Add(gcnew System::Tuple<System::Windows::Point, System::Windows::Point>(System::Windows::Point(line.first.x, line.first.y), System::Windows::Point(line.second.x, line.second.y))); lines->Add(gcnew System::Tuple<System::Windows::Point, System::Windows::Point>(System::Windows::Point(line.first.x, line.first.y), System::Windows::Point(line.second.x, line.second.y)));
@ -354,7 +356,7 @@ namespace CppInterop {
return lines; return lines;
} }
void DrawBox(System::Collections::Generic::List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) { void DrawBox(List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) {
cv::Scalar color = cv::Scalar(r,g,b,1); cv::Scalar color = cv::Scalar(r,g,b,1);
vector<pair<cv::Point, cv::Point>> vecLines; vector<pair<cv::Point, cv::Point>> vecLines;
@ -378,9 +380,9 @@ namespace CppInterop {
} }
// Getting the non-rigid shape parameters describing the facial expression // Getting the non-rigid shape parameters describing the facial expression
System::Collections::Generic::List<double>^ GetNonRigidParams() List<double>^ GetNonRigidParams()
{ {
auto non_rigid_params = gcnew System::Collections::Generic::List<double>(); auto non_rigid_params = gcnew List<double>();
for (int i = 0; i < clnf->params_local.rows; ++i) for (int i = 0; i < clnf->params_local.rows; ++i)
{ {
@ -391,9 +393,9 @@ namespace CppInterop {
} }
// Getting the rigid shape parameters describing face scale rotation and translation (scale,rotx,roty,rotz,tx,ty) // Getting the rigid shape parameters describing face scale rotation and translation (scale,rotx,roty,rotz,tx,ty)
System::Collections::Generic::List<double>^ GetRigidParams() List<double>^ GetRigidParams()
{ {
auto rigid_params = gcnew System::Collections::Generic::List<double>(); auto rigid_params = gcnew List<double>();
for (size_t i = 0; i < 6; ++i) for (size_t i = 0; i < 6; ++i)
{ {
@ -403,7 +405,7 @@ namespace CppInterop {
} }
// Rigid params followed by non-rigid ones // Rigid params followed by non-rigid ones
System::Collections::Generic::List<double>^ GetParams() List<double>^ GetParams()
{ {
auto all_params = GetRigidParams(); auto all_params = GetRigidParams();
all_params->AddRange(GetNonRigidParams()); all_params->AddRange(GetNonRigidParams());