diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 2b8b872..b138acb 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -234,9 +234,6 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, const FaceAnalysis::FaceAnalyser& face_analyser); -void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic); - - int main (int argc, char **argv) { @@ -498,9 +495,7 @@ int main (int argc, char **argv) catch(cv::Exception e) { WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - - + } } int frame_count = 0; @@ -700,7 +695,7 @@ int main (int argc, char **argv) if(output_files.size() > 0 && output_AUs) { cout << "Postprocessing the Action Unit predictions" << endl; - post_process_output_file(face_analyser, output_files[f_n], dynamic); + face_analyser.PostprocessOutputFile(output_files[f_n], dynamic); } // Reset the models for the next video face_analyser.Reset(); @@ -724,124 +719,6 @@ int main (int argc, char **argv) return 0; } -// Allows for post processing of the AU signal -void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic) -{ - - vector certainties; - vector successes; - vector timestamps; - vector>> predictions_reg; - vector>> predictions_class; - - // Construct the new values to overwrite the output file with - face_analyser.ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic); - face_analyser.ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic); - - int num_class = predictions_class.size(); - int num_reg = predictions_reg.size(); - - // Extract the indices of writing out first - vector au_reg_names = face_analyser.GetAURegNames(); - std::sort(au_reg_names.begin(), au_reg_names.end()); - vector inds_reg; - - // write out ar the correct index - for (string au_name : au_reg_names) - { - for (int i = 0; i < num_reg; ++i) - { - if (au_name.compare(predictions_reg[i].first) == 0) - { - inds_reg.push_back(i); - break; - } - } - } - - vector au_class_names = face_analyser.GetAUClassNames(); - std::sort(au_class_names.begin(), au_class_names.end()); - vector inds_class; - - // write out ar the correct index - for (string au_name : au_class_names) - { - for (int i = 0; i < num_class; ++i) - { - if (au_name.compare(predictions_class[i].first) == 0) - { - inds_class.push_back(i); - break; - } - } - } - // Read all of the output file in - vector output_file_contents; - - std::ifstream infile(output_file); - string line; - - while (std::getline(infile, line)) - output_file_contents.push_back(line); - - infile.close(); - - // Read the header and find all _r and _c parts in a file and use their indices - std::vector tokens; - boost::split(tokens, output_file_contents[0], boost::is_any_of(",")); - - int begin_ind = -1; - - for (size_t i = 0; i < tokens.size(); ++i) - { - if (tokens[i].find("AU") != string::npos && begin_ind == -1) - { - begin_ind = i; - break; - } - } - int end_ind = begin_ind + num_class + num_reg; - - // Now overwrite the whole file - std::ofstream outfile(output_file, ios_base::out); - // Write the header - outfile << std::setprecision(4); - outfile << output_file_contents[0].c_str() << endl; - - // Write the contents - for (int i = 1; i < (int)output_file_contents.size(); ++i) - { - std::vector tokens; - boost::split(tokens, output_file_contents[i], boost::is_any_of(",")); - - boost::trim(tokens[0]); - outfile << tokens[0]; - - for (int t = 1; t < (int)tokens.size(); ++t) - { - if (t >= begin_ind && t < end_ind) - { - if(t - begin_ind < num_reg) - { - outfile << ", " << predictions_reg[inds_reg[t - begin_ind]].second[i - 1]; - } - else - { - outfile << ", " << predictions_class[inds_class[t - begin_ind - num_reg]].second[i - 1]; - } - } - else - { - boost::trim(tokens[t]); - outfile << ", " << tokens[t]; - } - } - outfile << endl; - } - - -} - void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, int num_landmarks, int num_eye_lmks, int num_model_modes, vector au_names_class, vector au_names_reg) diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index d2e0388..a4518c3 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -420,7 +420,7 @@ namespace OpenFaceOffline // Setup the recorder first recorder = new Recorder(record_root, output_file_name, capture.width, capture.height, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, - RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy); + RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy, DynamicAUModels); int frame_id = 0; diff --git a/gui/OpenFaceOffline/Recorder.cs b/gui/OpenFaceOffline/Recorder.cs index cbf0474..adfa713 100644 --- a/gui/OpenFaceOffline/Recorder.cs +++ b/gui/OpenFaceOffline/Recorder.cs @@ -20,9 +20,13 @@ namespace OpenFaceOffline List au_reg_names; List au_class_names; + String out_filename; + + bool dynamic_AU_model; + public Recorder(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG, - CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy) + CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model) { this.output_2D_landmarks = output_2D_landmarks; this.output_3D_landmarks = output_3D_landmarks; @@ -32,12 +36,14 @@ namespace OpenFaceOffline this.fx = fx; this.fy = fy; this.cx = cx; this.cy = cy; + this.dynamic_AU_model = dynamic_AU_model; + if (!System.IO.Directory.Exists(root)) { System.IO.Directory.CreateDirectory(root); } - - output_features_file = new StreamWriter(root + "/" + filename + ".txt"); + out_filename = root + "/" + filename + ".csv"; + output_features_file = new StreamWriter(out_filename); output_features_file.Write("frame, timestamp, confidence, success"); if (output_gaze) @@ -216,7 +222,8 @@ namespace OpenFaceOffline if (record_HOG) face_analyser.StopHOGRecording(); - + + face_analyser.PostProcessOutputFile(out_filename, dynamic_AU_model); } } diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index 4c38beb..624a1fc 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -260,6 +260,11 @@ public: tracked_vid_writer->write(*tracked_face); } + void PostProcessOutputFile(System::String^ file, bool dynamic) + { + face_analyser->PostprocessOutputFile(msclr::interop::marshal_as(file), dynamic); + } + void AddNextFrame(OpenCVWrappers::RawImage^ frame, CppInterop::LandmarkDetector::CLNF^ clnf, double fx, double fy, double cx, double cy, bool online, bool vis_hog, bool vis_tracked) { face_analyser->AddNextFrame(frame->Mat, *clnf->getCLNF(), 0, online, vis_hog); diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index c9df566..7b5feaf 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -126,6 +126,9 @@ public: void ExtractAllPredictionsOfflineReg(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); void ExtractAllPredictionsOfflineClass(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); + // Helper function for post-processing AU output files + void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic); + private: // Where the predictions are kept diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index c75be7f..cfff07e 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -437,7 +437,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL std::vector> AU_predictions_reg_corrected; if(online) { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success); + AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success, true); } // Add the reg predictions to the historic data @@ -1255,4 +1255,122 @@ void FaceAnalyser::ReadRegressor(std::string fname, const vector& au_nam double FaceAnalyser::GetCurrentTimeSeconds() { return current_time_seconds; -} \ No newline at end of file +} + +// Allows for post processing of the AU signal +void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) +{ + + vector certainties; + vector successes; + vector timestamps; + vector>> predictions_reg; + vector>> predictions_class; + + // Construct the new values to overwrite the output file with + ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic); + ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic); + + int num_class = predictions_class.size(); + int num_reg = predictions_reg.size(); + + // Extract the indices of writing out first + vector au_reg_names = GetAURegNames(); + std::sort(au_reg_names.begin(), au_reg_names.end()); + vector inds_reg; + + // write out ar the correct index + for (string au_name : au_reg_names) + { + for (int i = 0; i < num_reg; ++i) + { + if (au_name.compare(predictions_reg[i].first) == 0) + { + inds_reg.push_back(i); + break; + } + } + } + + vector au_class_names = GetAUClassNames(); + std::sort(au_class_names.begin(), au_class_names.end()); + vector inds_class; + + // write out ar the correct index + for (string au_name : au_class_names) + { + for (int i = 0; i < num_class; ++i) + { + if (au_name.compare(predictions_class[i].first) == 0) + { + inds_class.push_back(i); + break; + } + } + } + // Read all of the output file in + vector output_file_contents; + + std::ifstream infile(output_file); + string line; + + while (std::getline(infile, line)) + output_file_contents.push_back(line); + + infile.close(); + + // Read the header and find all _r and _c parts in a file and use their indices + std::vector tokens; + boost::split(tokens, output_file_contents[0], boost::is_any_of(",")); + + int begin_ind = -1; + + for (size_t i = 0; i < tokens.size(); ++i) + { + if (tokens[i].find("AU") != string::npos && begin_ind == -1) + { + begin_ind = i; + break; + } + } + int end_ind = begin_ind + num_class + num_reg; + + // Now overwrite the whole file + std::ofstream outfile(output_file, ios_base::out); + // Write the header + outfile << std::setprecision(4); + outfile << output_file_contents[0].c_str() << endl; + + // Write the contents + for (int i = 1; i < (int)output_file_contents.size(); ++i) + { + std::vector tokens; + boost::split(tokens, output_file_contents[i], boost::is_any_of(",")); + + boost::trim(tokens[0]); + outfile << tokens[0]; + + for (int t = 1; t < (int)tokens.size(); ++t) + { + if (t >= begin_ind && t < end_ind) + { + if (t - begin_ind < num_reg) + { + outfile << ", " << predictions_reg[inds_reg[t - begin_ind]].second[i - 1]; + } + else + { + outfile << ", " << predictions_class[inds_class[t - begin_ind - num_reg]].second[i - 1]; + } + } + else + { + boost::trim(tokens[t]); + outfile << ", " << tokens[t]; + } + } + outfile << endl; + } + + +} diff --git a/matlab_runners/Feature Point Experiments/results/fps_yt.mat b/matlab_runners/Feature Point Experiments/results/fps_yt.mat index d45e40e..eab0cb3 100644 Binary files a/matlab_runners/Feature Point Experiments/results/fps_yt.mat and b/matlab_runners/Feature Point Experiments/results/fps_yt.mat differ diff --git a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf index e497391..c937405 100644 Binary files a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf and b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf differ diff --git a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat index 54be5be..4f8569d 100644 Binary files a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat and b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat differ diff --git a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat index a00dc8e..dd4e924 100644 Binary files a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat and b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat differ diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat index d725d13..4230754 100644 Binary files a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat and b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat differ