r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Full face model.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2016-06-03 16:53:27 -04:00
parent b75f12f078
commit 65e91da3a6
3 changed files with 78 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
exe/FeatureExtraction/FeatureExtraction.cpp
View file

@ -153,7 +153,7 @@ void create_directory(string output_path)
} }
void get_output_feature_params(vector<string> &output_similarity_aligned, vector<string> &output_hog_aligned_files, double &similarity_scale, void get_output_feature_params(vector<string> &output_similarity_aligned, vector<string> &output_hog_aligned_files, double &similarity_scale,
int &similarity_size, bool &grayscale, bool &rigid, bool& verbose, int &similarity_size, bool &grayscale, bool &rigid, bool& verbose, bool& dynamic,
bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze,
vector<string> &arguments); vector<string> &arguments);
@ -235,7 +235,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy,
const FaceAnalysis::FaceAnalyser& face_analyser); const FaceAnalysis::FaceAnalyser& face_analyser);
void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file); void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic);
int main (int argc, char **argv) int main (int argc, char **argv)
@ -306,6 +306,7 @@ int main (int argc, char **argv)
bool grayscale = false; bool grayscale = false;
bool video_output = false; bool video_output = false;
bool rigid = false; bool rigid = false;
bool dynamic = true; // Indicates if a dynamic AU model should be used (dynamic is useful if the video is long enough to include neutral expressions)
int num_hog_rows; int num_hog_rows;
int num_hog_cols; int num_hog_cols;
@ -318,7 +319,7 @@ int main (int argc, char **argv)
bool output_AUs = true; bool output_AUs = true;
bool output_gaze = true; bool output_gaze = true;
get_output_feature_params(output_similarity_align, output_hog_align_files, sim_scale, sim_size, grayscale, rigid, verbose, get_output_feature_params(output_similarity_align, output_hog_align_files, sim_scale, sim_size, grayscale, rigid, verbose, dynamic,
output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, arguments); output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, arguments);
// Used for image masking // Used for image masking
@ -359,13 +360,24 @@ int main (int argc, char **argv)
int curr_img = -1; int curr_img = -1;
string au_loc; string au_loc;
if(boost::filesystem::exists(path("AU_predictors/AU_all_best.txt")))
string au_loc_local;
if (dynamic)
{ {
au_loc = "AU_predictors/AU_all_best.txt"; au_loc_local = "AU_predictors/AU_all_best.txt";
} }
else else
{ {
path loc = path(arguments[0]).parent_path() / "AU_predictors/AU_all_best.txt"; au_loc_local = "AU_predictors/AU_all_static.txt";
}
if(boost::filesystem::exists(path(au_loc_local)))
{
au_loc = au_loc_local;
}
else
{
path loc = path(arguments[0]).parent_path() / au_loc_local;
if(exists(loc)) if(exists(loc))
{ {
@ -684,16 +696,10 @@ int main (int argc, char **argv)
output_file.close(); output_file.close();
if(output_files.size() > 0) if(output_files.size() > 0 && output_AUs)
{ {
cout << "Postprocessing the Action Unit predictions" << endl;
// If the video is long enough post-process it for AUs post_process_output_file(face_analyser, output_files[f_n], dynamic);
if (output_AUs && frame_count > 1000)
{
cout << "Postprocessing the Action Unit predictions" << endl;
post_process_output_file(face_analyser, output_files[f_n]);
}
} }
// Reset the models for the next video // Reset the models for the next video
face_analyser.Reset(); face_analyser.Reset();
@ -718,7 +724,7 @@ int main (int argc, char **argv)
} }
// Allows for post processing of the AU signal // Allows for post processing of the AU signal
void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file) void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic)
{ {
vector<double> certainties; vector<double> certainties;
@ -728,8 +734,8 @@ void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string
vector<std::pair<std::string, vector<double>>> predictions_class; vector<std::pair<std::string, vector<double>>> predictions_class;
// Construct the new values to overwrite the output file with // Construct the new values to overwrite the output file with
face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps); face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic);
face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps); face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic);
int num_class = predictions_class.size(); int num_class = predictions_class.size();
int num_reg = predictions_reg.size(); int num_reg = predictions_reg.size();
@ -1024,7 +1030,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo
void get_output_feature_params(vector<string> &output_similarity_aligned, vector<string> &output_hog_aligned_files, double &similarity_scale, void get_output_feature_params(vector<string> &output_similarity_aligned, vector<string> &output_hog_aligned_files, double &similarity_scale,
int &similarity_size, bool &grayscale, bool &rigid, bool& verbose, int &similarity_size, bool &grayscale, bool &rigid, bool& verbose, bool& dynamic,
bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze,
vector<string> &arguments) vector<string> &arguments)
{ {
@ -1041,6 +1047,9 @@ void get_output_feature_params(vector<string> &output_similarity_aligned, vector
string input_root = ""; string input_root = "";
string output_root = ""; string output_root = "";
// By default the model is dynamic
dynamic = true;
// First check if there is a root argument (so that videos and outputs could be defined more easilly) // First check if there is a root argument (so that videos and outputs could be defined more easilly)
for (size_t i = 0; i < arguments.size(); ++i) for (size_t i = 0; i < arguments.size(); ++i)
{ {
@ -1088,6 +1097,10 @@ void get_output_feature_params(vector<string> &output_similarity_aligned, vector
{ {
rigid = true; rigid = true;
} }
else if (arguments[i].compare("-au_static") == 0)
{
dynamic = false;
}
else if (arguments[i].compare("-g") == 0) else if (arguments[i].compare("-g") == 0)
{ {
grayscale = true; grayscale = true;

44
lib/local/FaceAnalyser/src/FaceAnalyser.cpp
View file

@ -592,10 +592,12 @@ void FaceAnalyser::PostprocessPredictions()
} }
} }
void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps) void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps, bool dynamic)
{ {
if(dynamic)
PostprocessPredictions(); {
PostprocessPredictions();
}
timestamps = this->timestamps; timestamps = this->timestamps;
au_predictions.clear(); au_predictions.clear();
@ -624,14 +626,14 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string,
} }
if(!au_good.empty()) if(au_good.empty() || !dynamic)
{ {
std::sort(au_good.begin(), au_good.end()); offsets.push_back(0.0);
offsets.push_back(au_good.at((int)au_good.size()/4));
} }
else else
{ {
offsets.push_back(0.0); std::sort(au_good.begin(), au_good.end());
offsets.push_back(au_good.at((int)au_good.size() / 4));
} }
aus_valid.push_back(au_good); aus_valid.push_back(au_good);
} }
@ -649,7 +651,7 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string,
au_predictions[au].second[frame] = (au_predictions[au].second[frame] - offsets[au]) * scaling; au_predictions[au].second[frame] = (au_predictions[au].second[frame] - offsets[au]) * scaling;
if(au_predictions[au].second[frame] < 0.5) if(au_predictions[au].second[frame] < 0.0)
au_predictions[au].second[frame] = 0; au_predictions[au].second[frame] = 0;
if(au_predictions[au].second[frame] > 5) if(au_predictions[au].second[frame] > 5)
@ -666,9 +668,12 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string,
} }
void FaceAnalyser::ExtractAllPredictionsOfflineClass(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps) void FaceAnalyser::ExtractAllPredictionsOfflineClass(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps, bool dynamic)
{ {
PostprocessPredictions(); if (dynamic)
{
PostprocessPredictions();
}
timestamps = this->timestamps; timestamps = this->timestamps;
au_predictions.clear(); au_predictions.clear();
@ -678,6 +683,25 @@ void FaceAnalyser::ExtractAllPredictionsOfflineClass(vector<std::pair<std::strin
string au_name = au_iter->first; string au_name = au_iter->first;
vector<double> au_vals = au_iter->second; vector<double> au_vals = au_iter->second;
// Perform a moving average of 7 frames on classifications
int window_size = 7;
vector<double> au_vals_tmp = au_vals;
for (size_t i = (window_size - 1)/2; i < au_vals.size() - (window_size - 1) / 2; ++i)
{
double sum = 0;
for (int w = -(window_size - 1) / 2; w < (window_size - 1) / 2; ++w)
{
sum += au_vals_tmp[i + w];
}
sum = sum / window_size;
if (sum < 0.5)
sum = 0;
else
sum = 1;
au_vals[i] = sum;
}
au_predictions.push_back(std::pair<string,vector<double>>(au_name, au_vals)); au_predictions.push_back(std::pair<string,vector<double>>(au_name, au_vals));
} }

24
lib/local/FaceAnalyser/src/Face_utils.cpp
View file

@ -221,19 +221,19 @@ namespace FaceAnalysis
destination_landmarks.col(1) = destination_landmarks.col(1) + warp_matrix(1,2); destination_landmarks.col(1) = destination_landmarks.col(1) + warp_matrix(1,2);
// Move the eyebrows up to include more of upper face // Move the eyebrows up to include more of upper face
destination_landmarks.at<double>(0,1) -= 15; destination_landmarks.at<double>(0,1) -= 30;
destination_landmarks.at<double>(16,1) -= 15; destination_landmarks.at<double>(16,1) -= 30;
destination_landmarks.at<double>(17,1) -= 7; destination_landmarks.at<double>(17,1) -= 30;
destination_landmarks.at<double>(18,1) -= 7; destination_landmarks.at<double>(18,1) -= 30;
destination_landmarks.at<double>(19,1) -= 7; destination_landmarks.at<double>(19,1) -= 30;
destination_landmarks.at<double>(20,1) -= 7; destination_landmarks.at<double>(20,1) -= 30;
destination_landmarks.at<double>(21,1) -= 7; destination_landmarks.at<double>(21,1) -= 30;
destination_landmarks.at<double>(22,1) -= 7; destination_landmarks.at<double>(22,1) -= 30;
destination_landmarks.at<double>(23,1) -= 7; destination_landmarks.at<double>(23,1) -= 30;
destination_landmarks.at<double>(24,1) -= 7; destination_landmarks.at<double>(24,1) -= 30;
destination_landmarks.at<double>(25,1) -= 7; destination_landmarks.at<double>(25,1) -= 30;
destination_landmarks.at<double>(26,1) -= 7; destination_landmarks.at<double>(26,1) -= 30;
destination_landmarks = cv::Mat(destination_landmarks.t()).reshape(1, 1).t(); destination_landmarks = cv::Mat(destination_landmarks.t()).reshape(1, 1).t();