From 29d13878da52a61642b76d298b765ef233a2d889 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Fri, 9 Dec 2016 10:06:04 -0500 Subject: [PATCH] Towards AU post-processing in the GUI. --- exe/FeatureExtraction/FeatureExtraction.cpp | 127 +----------------- gui/OpenFaceOffline/MainWindow.xaml.cs | 2 +- gui/OpenFaceOffline/Recorder.cs | 15 ++- lib/local/CppInerop/FaceAnalyserInterop.h | 5 + lib/local/FaceAnalyser/include/FaceAnalyser.h | 3 + lib/local/FaceAnalyser/src/FaceAnalyser.cpp | 122 ++++++++++++++++- .../results/fps_yt.mat | Bin 4216623 -> 4216621 bytes .../results/in-the-wild-res-no-outline.pdf | Bin 10497 -> 10497 bytes .../results/landmark_detections.mat | Bin 63352 -> 63352 bytes .../Gaze Experiments/mpii_1500_errs.mat | Bin 11745 -> 11745 bytes .../Head Pose Experiments/results/Pose_OF.mat | Bin 2689437 -> 2689437 bytes 11 files changed, 142 insertions(+), 132 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 2b8b872..b138acb 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -234,9 +234,6 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, const FaceAnalysis::FaceAnalyser& face_analyser); -void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic); - - int main (int argc, char **argv) { @@ -498,9 +495,7 @@ int main (int argc, char **argv) catch(cv::Exception e) { WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - - + } } int frame_count = 0; @@ -700,7 +695,7 @@ int main (int argc, char **argv) if(output_files.size() > 0 && output_AUs) { cout << "Postprocessing the Action Unit predictions" << endl; - post_process_output_file(face_analyser, output_files[f_n], dynamic); + face_analyser.PostprocessOutputFile(output_files[f_n], dynamic); } // Reset the models for the next video face_analyser.Reset(); @@ -724,124 +719,6 @@ int main (int argc, char **argv) return 0; } -// Allows for post processing of the AU signal -void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic) -{ - - vector certainties; - vector successes; - vector timestamps; - vector>> predictions_reg; - vector>> predictions_class; - - // Construct the new values to overwrite the output file with - face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic); - face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic); - - int num_class = predictions_class.size(); - int num_reg = predictions_reg.size(); - - // Extract the indices of writing out first - vector au_reg_names = face_analyser.GetAURegNames(); - std::sort(au_reg_names.begin(), au_reg_names.end()); - vector inds_reg; - - // write out ar the correct index - for (string au_name : au_reg_names) - { - for (int i = 0; i < num_reg; ++i) - { - if (au_name.compare(predictions_reg[i].first) == 0) - { - inds_reg.push_back(i); - break; - } - } - } - - vector au_class_names = face_analyser.GetAUClassNames(); - std::sort(au_class_names.begin(), au_class_names.end()); - vector inds_class; - - // write out ar the correct index - for (string au_name : au_class_names) - { - for (int i = 0; i < num_class; ++i) - { - if (au_name.compare(predictions_class[i].first) == 0) - { - inds_class.push_back(i); - break; - } - } - } - // Read all of the output file in - vector output_file_contents; - - std::ifstream infile(output_file); - string line; - - while (std::getline(infile, line)) - output_file_contents.push_back(line); - - infile.close(); - - // Read the header and find all _r and _c parts in a file and use their indices - std::vector tokens; - boost::split(tokens, output_file_contents[0], boost::is_any_of(",")); - - int begin_ind = -1; - - for (size_t i = 0; i < tokens.size(); ++i) - { - if (tokens[i].find("AU") != string::npos && begin_ind == -1) - { - begin_ind = i; - break; - } - } - int end_ind = begin_ind + num_class + num_reg; - - // Now overwrite the whole file - std::ofstream outfile(output_file, ios_base::out); - // Write the header - outfile << std::setprecision(4); - outfile << output_file_contents[0].c_str() << endl; - - // Write the contents - for (int i = 1; i < (int)output_file_contents.size(); ++i) - { - std::vector tokens; - boost::split(tokens, output_file_contents[i], boost::is_any_of(",")); - - boost::trim(tokens[0]); - outfile << tokens[0]; - - for (int t = 1; t < (int)tokens.size(); ++t) - { - if (t >= begin_ind && t < end_ind) - { - if(t - begin_ind < num_reg) - { - outfile << ", " << predictions_reg[inds_reg[t - begin_ind]].second[i - 1]; - } - else - { - outfile << ", " << predictions_class[inds_class[t - begin_ind - num_reg]].second[i - 1]; - } - } - else - { - boost::trim(tokens[t]); - outfile << ", " << tokens[t]; - } - } - outfile << endl; - } - - -} - void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, int num_landmarks, int num_eye_lmks, int num_model_modes, vector au_names_class, vector au_names_reg) diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index d2e0388..a4518c3 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -420,7 +420,7 @@ namespace OpenFaceOffline // Setup the recorder first recorder = new Recorder(record_root, output_file_name, capture.width, capture.height, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, - RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy); + RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy, DynamicAUModels); int frame_id = 0; diff --git a/gui/OpenFaceOffline/Recorder.cs b/gui/OpenFaceOffline/Recorder.cs index cbf0474..adfa713 100644 --- a/gui/OpenFaceOffline/Recorder.cs +++ b/gui/OpenFaceOffline/Recorder.cs @@ -20,9 +20,13 @@ namespace OpenFaceOffline List au_reg_names; List au_class_names; + String out_filename; + + bool dynamic_AU_model; + public Recorder(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG, - CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy) + CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model) { this.output_2D_landmarks = output_2D_landmarks; this.output_3D_landmarks = output_3D_landmarks; @@ -32,12 +36,14 @@ namespace OpenFaceOffline this.fx = fx; this.fy = fy; this.cx = cx; this.cy = cy; + this.dynamic_AU_model = dynamic_AU_model; + if (!System.IO.Directory.Exists(root)) { System.IO.Directory.CreateDirectory(root); } - - output_features_file = new StreamWriter(root + "/" + filename + ".txt"); + out_filename = root + "/" + filename + ".csv"; + output_features_file = new StreamWriter(out_filename); output_features_file.Write("frame, timestamp, confidence, success"); if (output_gaze) @@ -216,7 +222,8 @@ namespace OpenFaceOffline if (record_HOG) face_analyser.StopHOGRecording(); - + + face_analyser.PostProcessOutputFile(out_filename, dynamic_AU_model); } } diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index 4c38beb..624a1fc 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -260,6 +260,11 @@ public: tracked_vid_writer->write(*tracked_face); } + void PostProcessOutputFile(System::String^ file, bool dynamic) + { + face_analyser->PostprocessOutputFile(msclr::interop::marshal_as(file), dynamic); + } + void AddNextFrame(OpenCVWrappers::RawImage^ frame, CppInterop::LandmarkDetector::CLNF^ clnf, double fx, double fy, double cx, double cy, bool online, bool vis_hog, bool vis_tracked) { face_analyser->AddNextFrame(frame->Mat, *clnf->getCLNF(), 0, online, vis_hog); diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index c9df566..7b5feaf 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -126,6 +126,9 @@ public: void ExtractAllPredictionsOfflineReg(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); void ExtractAllPredictionsOfflineClass(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); + // Helper function for post-processing AU output files + void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic); + private: // Where the predictions are kept diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index c75be7f..cfff07e 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -437,7 +437,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL std::vector> AU_predictions_reg_corrected; if(online) { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success); + AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success, true); } // Add the reg predictions to the historic data @@ -1255,4 +1255,122 @@ void FaceAnalyser::ReadRegressor(std::string fname, const vector& au_nam double FaceAnalyser::GetCurrentTimeSeconds() { return current_time_seconds; -} \ No newline at end of file +} + +// Allows for post processing of the AU signal +void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) +{ + + vector certainties; + vector successes; + vector timestamps; + vector>> predictions_reg; + vector>> predictions_class; + + // Construct the new values to overwrite the output file with + ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic); + ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic); + + int num_class = predictions_class.size(); + int num_reg = predictions_reg.size(); + + // Extract the indices of writing out first + vector au_reg_names = GetAURegNames(); + std::sort(au_reg_names.begin(), au_reg_names.end()); + vector inds_reg; + + // write out ar the correct index + for (string au_name : au_reg_names) + { + for (int i = 0; i < num_reg; ++i) + { + if (au_name.compare(predictions_reg[i].first) == 0) + { + inds_reg.push_back(i); + break; + } + } + } + + vector au_class_names = GetAUClassNames(); + std::sort(au_class_names.begin(), au_class_names.end()); + vector inds_class; + + // write out ar the correct index + for (string au_name : au_class_names) + { + for (int i = 0; i < num_class; ++i) + { + if (au_name.compare(predictions_class[i].first) == 0) + { + inds_class.push_back(i); + break; + } + } + } + // Read all of the output file in + vector output_file_contents; + + std::ifstream infile(output_file); + string line; + + while (std::getline(infile, line)) + output_file_contents.push_back(line); + + infile.close(); + + // Read the header and find all _r and _c parts in a file and use their indices + std::vector tokens; + boost::split(tokens, output_file_contents[0], boost::is_any_of(",")); + + int begin_ind = -1; + + for (size_t i = 0; i < tokens.size(); ++i) + { + if (tokens[i].find("AU") != string::npos && begin_ind == -1) + { + begin_ind = i; + break; + } + } + int end_ind = begin_ind + num_class + num_reg; + + // Now overwrite the whole file + std::ofstream outfile(output_file, ios_base::out); + // Write the header + outfile << std::setprecision(4); + outfile << output_file_contents[0].c_str() << endl; + + // Write the contents + for (int i = 1; i < (int)output_file_contents.size(); ++i) + { + std::vector tokens; + boost::split(tokens, output_file_contents[i], boost::is_any_of(",")); + + boost::trim(tokens[0]); + outfile << tokens[0]; + + for (int t = 1; t < (int)tokens.size(); ++t) + { + if (t >= begin_ind && t < end_ind) + { + if (t - begin_ind < num_reg) + { + outfile << ", " << predictions_reg[inds_reg[t - begin_ind]].second[i - 1]; + } + else + { + outfile << ", " << predictions_class[inds_class[t - begin_ind - num_reg]].second[i - 1]; + } + } + else + { + boost::trim(tokens[t]); + outfile << ", " << tokens[t]; + } + } + outfile << endl; + } + + +} diff --git a/matlab_runners/Feature Point Experiments/results/fps_yt.mat b/matlab_runners/Feature Point Experiments/results/fps_yt.mat index d45e40ec314be4dc6a9e0f96a9efd82195bededc..eab0cb3d8fba5b881a5753908e3b7024737482e0 100644 GIT binary patch delta 859 zcmWO3Z7|aT9KiAYu|`x1Bhgl@m51hGLz|{~sD!Jz7R9C;9r8kiCbaZ8PdB^NdXUQL zVa&Wp_m8#7q2!iZ%0qQ|Xys|gJTzAx`@Q(Q|K5EY6on@Bpq%P(JSCeLdM1(R6+rY3 zpnLn#ss2PCFK<5y+eLC3E6J*ol0--_$}UCAzcEA=hla^W%CfnG&XY#RmbW_AFYL1OlyjeyjL$Od% zk>(a&i;r=}W>~Y1;m9m)@5k1T-cKKd!7qpcY5S?_zmmSTw;8!HgUUg%PGW@_Zp2-- zeWd&=GF=N_mvLVl>lONj?=v~v{J_kDg4f&R$JPW3!+1rn2jk_+FC#A}Dk2%N=vlk+ z5rgG$K}%>#5Ia5KZpUZ)cI#29m7OLz@<`1UXZ~%6SF>H;DwGo!v&Yo3?}S=+eo9jE zMCOUY!vxkrBo#{|U$Qrzj!=Ce8~L8bYwuS>mP2^-)ofm%z$W({Wg!Bm7w6#^zHvI= zPeLxI<9ygm1K0RMAwBs|aWv#mRR7cT?jT2ci|C2ZlyZ(G=X6Du3}@cH_a_QZT}w#L zr;%z6(@pt9Y@M7t6#tUYIZLZv+-3N^)J@?v>gyKOwYCW=%sjgwQDrfqR+i}(78Rp3 z?sVNqH8<{=pe;$0MgMWZXcElkCI}tp44&ia)P9bUJHOULYrS5D292(e`z6|cVWrx} zk*1wkX`04$K!KM{satcjoU)&8)B4$>4+3)TS~ zFb5W3J+K5;U<23)HUVp}8EgSIz!um6Jg^52U@LG0+kg{r2HODv5WxP;03&a5AX$4Km&fj9|VAXfDRZS5bOs`lvWwE*!&;F C(VK|? delta 855 zcmWO3?K9H>902g&=3yj}$couYWqI1hW~MQu)+Mw|spC{0!dyG$VIFrVQttB5!x7RO z56z|;_d^@kd3aIoFb_Lcso^lhixy{6-xr_1;3Jh4VLL!29UmsVOA1MhCs7$B8q<%? z@T0LvUR0V-70EWn+)sYn48CWI%BsN_H$F5TZ0aX^QGG6k`i)p ziJ5LK!R4P7A$FH(YrIk2`GQ;+qp@InQht8d7qXnfu*h8;vOFKpc}m{w5LLQ}LW-?( zYQ3~gy$(1U#eTF!6x|&>sIU=dDc?tDmk)4t&sqs=G}W$hVax5uePwjAKTk-<4>4>P ztdlMoXAW{7gmxD@rai~M=j8~7zhx?WpY%@>4oumx=o1zR{?fA^uT=Nsr8G&cR@sWv z==24V6xG!TQ9Xy-KN1=uZnU@jHK^F-Oo!6@xjhOd%Arv?Gaz8(1zh z{k${$deRUO?v$YuZ1rOZr*;^5WX4tZi)XoGj&`V(INlvV53)KCA0O z8fvaNr*5y1_;#q7y3y6BMeS%o7Q0hiCFseY-jH8kn4IxRkJ(%Ll1__{l1SBA<0xrW zgXe8%L?3_WbCBGFGkaDDf)G_^kUmnI_oys`g#|{y7?=Q4fCFa09P9%YU_Y<~2Y?l@ z1_yx+I0S5g9XJf^0UjIyM}Y%y1O(s&h=2r+f#bj#oB$`$4A!as+ww@Gk+w4L`cuFK ukilu-3fzD@@Bp5G0;qrnynr{L0|xK`OyCPxzz>`OY~T+9Q15!qPVawqu!H6R diff --git a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf index e497391bfbb2231bfc930ee7db083c30514d1b2e..c9374050b2d3918430c3680cbd70064bd4b2ae19 100644 GIT binary patch delta 90 zcmZn+Y7Cl?!){?{VQ6Y>y0KVQ!`af@(7@cy+|t#=(bCAx)Y!?wz{$|r&CJ=<(8$!) P*}zW0hLDoUw>8uN8Qm6* delta 90 zcmZn+Y7Cl?!)|J5Zen6!zOh(U!`al-(ZJl&(9+q}$->0N(9zMv%+<)$$;8pZ)y3J^ P+`>-5hLDoUw>8uNAo3Q< diff --git a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat index 54be5becaa2dfb32709b6aa99dc2390f9b8c678b..4f8569da1795dfece91775290fb4ebdd3f9ea4a5 100644 GIT binary patch delta 43 zcmezIj`_zs<_RVeAsM9#E~&{11{Mm27FLF)R>p=3Mh1pv69bheCa`WSVfhXKS3wS6 delta 43 zcmezIj`_zs<_RVezWI3yE~&{12Br#z=2j*qRt5$NMh1pv69bheCa`WSVfhXKRo@O$ diff --git a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat index a00dc8e311959786034c5cec64131eb61d2abc71..dd4e924899eb8a992e05df7328f6c5c1771d1d29 100644 GIT binary patch delta 41 vcmaDD{V;lhi9|?7se(&tvVwtyf{}rhv89!XC6HlgHZf3nVgl>Nl3ZN?9U2U* delta 41 wcmaDD{V;lhi9~p6ih@gOvVwuRf`O%#sil>HrGk-xq1nVh<%tQb8%uI^0UKZpp8x;= diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat index d725d13ac385ab5c71723c47b391c967e671e175..4230754a0448d90c5753ad41c84b162e940277ac 100644 GIT binary patch delta 287 zcmXBMy)Oe{0Knn*RdqeuJDiWUl($+RH}$U3s;-2^(nKOQNmCJX8wsP~R&NkfHz6^Q zPA9AqgGFMy-Tz>b_%()SeY%&ovWtt=^zKndZMK@qt0})$_txuWUzI(-sv8A;5bw2Z zX*t)=$3Lydr74DI7Qa8YONQt@n0K}iqH&{B zW;Uo;CWbMBBvPHq)$ delta 287 zcmbO`c?siOAf8|%;hUeQ;F6lGU|_0XXl`X}0YnN$28L!61C=Kxux>2jXXN``d}D9V z-=2PEhFSO6A2&~AY@f&o!c0KS48$xz%nHP8K+F!r9NQ-{a+dQ57ner;+Pw72X-0-s z2RPdJ85kG>r%&YNlxjES;{;+ZAm#>Q9w6ogVm=_|2Vwyr76f7;AQlE<5g--?Vlf~V z2V#lsrhJmePYYi*;|s63`}!FJ!|Yw`ZXn}*AjWInl5E$!B?ZLNKr92qvOp{c#PUF_ d0K|$wtOUf$K&%49sz9s;#Om8MZ)ue10{}9UUHkw5