r
/
sustaining_gazes
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Working towards coping without AU and eye models. Fixes to the visualizer.

This commit is contained in:
Tadas Baltrusaitis 2018-03-30 09:36:32 +01:00
parent 51fbf805bc
commit 6e15e81777
8 changed files with 139 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -95,3 +95,5 @@ lib/3rdParty/CameraEnumerator/Release/
lib/local/Utilities/Release/
exe/FaceLandmarkVidMulti/processed/
matlab_runners/Demos/processed/multi_face_aligned/
exe/releases/OpenFace_0.4.1_win_x64_landmarks/
exe/releases/OpenFace_*

16
exe/FaceLandmarkImg/FaceLandmarkImg.cpp
View File

@ -127,18 +127,30 @@ int main (int argc, char **argv)
captured_image = image_reader.GetNextImage();
if (!face_model.eye_model)
{
cout << "WARNING: no eye model found" << endl;
}
if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0)
{
cout << "WARNING: no Action Unit models found" << endl;
}
cout << "Starting tracking" << endl;
while (!captured_image.empty())
{
Utilities::RecorderOpenFaceParameters recording_params(arguments, false, false,
image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy);
if (!face_model.eye_model)
{
recording_params.setOutputGaze(false);
}
Utilities::RecorderOpenFace open_face_rec(image_reader.name, recording_params, arguments);
visualizer.SetImage(captured_image, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy);
if (recording_params.outputGaze() && !face_model.eye_model)
cout << "WARNING: no eye model defined, but outputting gaze" << endl;
// Making sure the image is in uchar grayscale
cv::Mat_<uchar> grayscale_image = image_reader.GetGrayFrame();

5
exe/FaceLandmarkVid/FaceLandmarkVid.cpp
View File

@ -104,6 +104,11 @@ int main (int argc, char **argv)
// The modules that are being used for tracking
LandmarkDetector::CLNF face_model(det_parameters.model_location);
if (!face_model.eye_model)
{
cout << "WARNING: no eye model found" << endl;
}
// Open a sequence
Utilities::SequenceCapture sequence_reader;

19
exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp
View File

@ -158,6 +158,16 @@ int main (int argc, char **argv)
face_analysis_params.OptimizeForImages();
FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
if (!face_model.eye_model)
{
cout << "WARNING: no eye model found" << endl;
}
if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0)
{
cout << "WARNING: no Action Unit models found" << endl;
}
// Open a sequence
Utilities::SequenceCapture sequence_reader;
@ -185,12 +195,13 @@ int main (int argc, char **argv)
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(),
sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps);
// Do not do AU detection on multi-face case as it is not supported
recording_params.setOutputAUs(false);
if (!face_model.eye_model)
{
recording_params.setOutputGaze(false);
}
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
if (recording_params.outputGaze() && !face_model.eye_model)
cout << "WARNING: no eye model defined, but outputting gaze" << endl;
if (sequence_reader.IsWebcam())
{

14
exe/FeatureExtraction/FeatureExtraction.cpp
View File

@ -121,6 +121,16 @@ int main (int argc, char **argv)
FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
if (!face_model.eye_model)
{
cout << "WARNING: no eye model found" << endl;
}
if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0)
{
cout << "WARNING: no Action Unit models found" << endl;
}
Utilities::SequenceCapture sequence_reader;
// A utility for visualizing the results
@ -150,6 +160,10 @@ int main (int argc, char **argv)
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(),
sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps);
if (!face_model.eye_model)
{
recording_params.setOutputGaze(false);
}
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
if (recording_params.outputGaze() && !face_model.eye_model)

60
gui/OpenFaceDemo/MainWindow.xaml.cs
View File

@ -259,37 +259,47 @@ namespace OpenFaceDemo
{
var au_regs = face_analyser.GetCurrentAUsReg();
if(au_regs.Count > 0)
{
double smile = (au_regs["AU12"] + au_regs["AU06"] + au_regs["AU25"]) / 13.0;
double frown = (au_regs["AU15"] + au_regs["AU17"]) / 12.0;
double smile = (au_regs["AU12"] + au_regs["AU06"] + au_regs["AU25"]) / 13.0;
double frown = (au_regs["AU15"] + au_regs["AU17"]) / 12.0;
double brow_up = (au_regs["AU01"] + au_regs["AU02"]) / 10.0;
double brow_down = au_regs["AU04"] / 5.0;
double brow_up = (au_regs["AU01"] + au_regs["AU02"]) / 10.0;
double brow_down = au_regs["AU04"] / 5.0;
double eye_widen = au_regs["AU05"] / 3.0;
double nose_wrinkle = au_regs["AU09"] / 4.0;
double eye_widen = au_regs["AU05"] / 3.0;
double nose_wrinkle = au_regs["AU09"] / 4.0;
Dictionary<int, double> smileDict = new Dictionary<int, double>();
smileDict[0] = 0.7 * smile_cumm + 0.3 * smile;
smileDict[1] = 0.7 * frown_cumm + 0.3 * frown;
smilePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = smileDict, Confidence = confidence });
Dictionary<int, double> smileDict = new Dictionary<int, double>();
smileDict[0] = 0.7 * smile_cumm + 0.3 * smile;
smileDict[1] = 0.7 * frown_cumm + 0.3 * frown;
smilePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = smileDict, Confidence = confidence });
Dictionary<int, double> browDict = new Dictionary<int, double>();
browDict[0] = 0.7 * brow_up_cumm + 0.3 * brow_up;
browDict[1] = 0.7 * brow_down_cumm + 0.3 * brow_down;
browPlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = browDict, Confidence = confidence });
Dictionary<int, double> browDict = new Dictionary<int, double>();
browDict[0] = 0.7 * brow_up_cumm + 0.3 * brow_up;
browDict[1] = 0.7 * brow_down_cumm + 0.3 * brow_down;
browPlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = browDict, Confidence = confidence });
Dictionary<int, double> eyeDict = new Dictionary<int, double>();
eyeDict[0] = 0.7 * widen_cumm + 0.3 * eye_widen;
eyeDict[1] = 0.7 * wrinkle_cumm + 0.3 * nose_wrinkle;
eyePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = eyeDict, Confidence = confidence });
Dictionary<int, double> eyeDict = new Dictionary<int, double>();
eyeDict[0] = 0.7 * widen_cumm + 0.3 * eye_widen;
eyeDict[1] = 0.7 * wrinkle_cumm + 0.3 * nose_wrinkle;
eyePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = eyeDict, Confidence = confidence });
smile_cumm = smileDict[0];
frown_cumm = smileDict[1];
brow_up_cumm = browDict[0];
brow_down_cumm = browDict[1];
widen_cumm = eyeDict[0];
wrinkle_cumm = eyeDict[1];
smile_cumm = smileDict[0];
frown_cumm = smileDict[1];
brow_up_cumm = browDict[0];
brow_down_cumm = browDict[1];
widen_cumm = eyeDict[0];
wrinkle_cumm = eyeDict[1];
}
else
{
// If no AUs present disable the AU visualization
MainGrid.ColumnDefinitions[2].Width = new GridLength(0);
eyePlot.Visibility = Visibility.Collapsed;
browPlot.Visibility = Visibility.Collapsed;
smilePlot.Visibility = Visibility.Collapsed;
}
Dictionary<int, double> poseDict = new Dictionary<int, double>();
poseDict[0] = -pose[3];

1
lib/local/Utilities/include/RecorderOpenFaceParameters.h
View File

@ -78,6 +78,7 @@ namespace Utilities
float getCy() const { return cy; }
void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; }
void setOutputGaze(bool output_gaze) { this->output_gaze = output_gaze; }
private:

59
lib/local/Utilities/src/Visualizer.cpp
View File

@ -34,6 +34,7 @@
#include <sstream>
#include <iomanip>
#include <map>
#include <set>
#include "Visualizer.h"
#include "VisualizationUtils.h"
@ -236,21 +237,69 @@ void Visualizer::SetObservationActionUnits(const std::vector<std::pair<std::stri
{
if(au_intensities.size() > 0 || au_occurences.size() > 0)
{
const int NB_AUS = 17;
std::set<std::string> au_names;
std::map<std::string, bool> occurences_map;
std::map<std::string, double> intensities_map;
for (size_t idx = 0; idx < au_intensities.size(); idx++)
{
au_names.insert(au_intensities[idx].first);
intensities_map[au_intensities[idx].first] = au_intensities[idx].second;
}
for (size_t idx = 0; idx < au_occurences.size(); idx++)
{
au_names.insert(au_occurences[idx].first);
occurences_map[au_occurences[idx].first] = au_occurences[idx].second;
}
const int AU_TRACKBAR_LENGTH = 400;
const int AU_TRACKBAR_HEIGHT = 10;
const int MARGIN_X = 185;
const int MARGIN_Y = 10;
action_units_image = cv::Mat(NB_AUS * (AU_TRACKBAR_HEIGHT + 10) + MARGIN_Y * 2, AU_TRACKBAR_LENGTH + MARGIN_X, CV_8UC3, cv::Scalar(255,255,255));
const int nb_aus = au_names.size();
// Do not reinitialize
if(action_units_image.empty())
{
action_units_image = cv::Mat(nb_aus * (AU_TRACKBAR_HEIGHT + 10) + MARGIN_Y * 2, AU_TRACKBAR_LENGTH + MARGIN_X, CV_8UC3, cv::Scalar(255,255,255));
}
else
{
action_units_image.setTo(255);
}
std::map<std::string, std::pair<bool, double>> aus;
// first, prepare a mapping "AU name" -> { present, intensity }
for (size_t idx = 0; idx < au_intensities.size(); idx++) {
aus[au_intensities[idx].first] = std::make_pair(au_occurences[idx].second != 0, au_intensities[idx].second);
for (auto au_name : au_names)
{
// Insert the intensity and AU presense (as these do not always overlap check if they exist first)
bool occurence = false;
if (occurences_map.find(au_name) != occurences_map.end())
{
occurence = occurences_map[au_name] != 0;
}
else
{
// If we do not have an occurence label, trust the intensity one
occurence = intensities_map[au_name] > 1;
}
double intensity = 0.0;
if (intensities_map.find(au_name) != intensities_map.end())
{
intensity = intensities_map[au_name];
}
else
{
// If we do not have an intensity label, trust the occurence one
intensity = occurences_map[au_name] == 0 ? 0 : 5;
}
aus[au_name] = std::make_pair(occurence, intensity);
}
// then, build the graph
@ -382,12 +431,10 @@ char Visualizer::ShowObservation()
}
if (vis_aus && !action_units_image.empty())
{
cv::namedWindow("action units", cv::WindowFlags::WINDOW_KEEPRATIO);
cv::imshow("action units", action_units_image);
}
if (vis_track)
{
cv::namedWindow("tracking result", cv::WindowFlags::WINDOW_KEEPRATIO);
cv::imshow("tracking result", captured_image);
}
return cv::waitKey(1);