diff --git a/OpenFace.sln b/OpenFace.sln index 9c19ba7..a57bf04 100644 --- a/OpenFace.sln +++ b/OpenFace.sln @@ -25,7 +25,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkImg", "exe\Face EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GazeAnalyser", "lib\local\GazeAnalyser\GazeAnalyser.vcxproj", "{5F915541-F531-434F-9C81-79F5DB58012B}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Recorder", "lib\local\Recorder\Recorder.vcxproj", "{8E741EA2-9386-4CF2-815E-6F9B08991EAC}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Utilities", "lib\local\Utilities\Utilities.vcxproj", "{8E741EA2-9386-4CF2-815E-6F9B08991EAC}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/exe/FeatureExtraction/FeatureExtraction.vcxproj b/exe/FeatureExtraction/FeatureExtraction.vcxproj index 0499c48..0b72e06 100644 --- a/exe/FeatureExtraction/FeatureExtraction.vcxproj +++ b/exe/FeatureExtraction/FeatureExtraction.vcxproj @@ -111,7 +111,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Recorder\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -127,7 +127,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Recorder\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -145,7 +145,7 @@ false true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Recorder\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed StreamingSIMDExtensions2 @@ -167,7 +167,7 @@ false true WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Recorder\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed AdvancedVectorExtensions @@ -194,7 +194,7 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} - + {8e741ea2-9386-4cf2-815e-6f9b08991eac} diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp index 2a3b90e..23d4698 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp @@ -488,6 +488,10 @@ double DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_ &image, FaceModelParameters& pa detection_certainty = landmark_validator.Check(orientation, image, detected_landmarks); - detection_success = detection_certainty < params.validation_boundary; + detection_success = detection_certainty > params.validation_boundary; } else { diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index 96aa02a..08910fe 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -253,7 +253,7 @@ void FaceModelParameters::init() reg_factor = 25; weight_factor = 0; // By default do not use NU-RLMS for videos as it does not work as well for them - validation_boundary = -0.45; + validation_boundary = 0.725; limit_pose = true; multi_view = false; diff --git a/lib/local/Recorder/Recorder.vcxproj b/lib/local/Utilities/Utilities.vcxproj similarity index 98% rename from lib/local/Recorder/Recorder.vcxproj rename to lib/local/Utilities/Utilities.vcxproj index 371b48b..f6ab3a8 100644 --- a/lib/local/Recorder/Recorder.vcxproj +++ b/lib/local/Utilities/Utilities.vcxproj @@ -21,8 +21,9 @@ {8E741EA2-9386-4CF2-815E-6F9B08991EAC} Win32Proj - Recorder + Utilities 8.1 + Utilities diff --git a/lib/local/Recorder/Recorder.vcxproj.filters b/lib/local/Utilities/Utilities.vcxproj.filters similarity index 100% rename from lib/local/Recorder/Recorder.vcxproj.filters rename to lib/local/Utilities/Utilities.vcxproj.filters diff --git a/lib/local/Recorder/include/RecorderCSV.h b/lib/local/Utilities/include/RecorderCSV.h similarity index 99% rename from lib/local/Recorder/include/RecorderCSV.h rename to lib/local/Utilities/include/RecorderCSV.h index 060f50f..1f5754a 100644 --- a/lib/local/Recorder/include/RecorderCSV.h +++ b/lib/local/Utilities/include/RecorderCSV.h @@ -42,7 +42,7 @@ // OpenCV includes #include -namespace Recorder +namespace Utilities { //=========================================================================== diff --git a/lib/local/Recorder/include/RecorderHOG.h b/lib/local/Utilities/include/RecorderHOG.h similarity index 99% rename from lib/local/Recorder/include/RecorderHOG.h rename to lib/local/Utilities/include/RecorderHOG.h index 06cae80..69a31ae 100644 --- a/lib/local/Recorder/include/RecorderHOG.h +++ b/lib/local/Utilities/include/RecorderHOG.h @@ -43,7 +43,7 @@ #include #include -namespace Recorder +namespace Utilities { //=========================================================================== diff --git a/lib/local/Recorder/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h similarity index 99% rename from lib/local/Recorder/include/RecorderOpenFace.h rename to lib/local/Utilities/include/RecorderOpenFace.h index dfe96c4..5ecac8a 100644 --- a/lib/local/Recorder/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -45,7 +45,7 @@ #include #include -namespace Recorder +namespace Utilities { //=========================================================================== diff --git a/lib/local/Recorder/include/RecorderOpenFaceParameters.h b/lib/local/Utilities/include/RecorderOpenFaceParameters.h similarity index 99% rename from lib/local/Recorder/include/RecorderOpenFaceParameters.h rename to lib/local/Utilities/include/RecorderOpenFaceParameters.h index 9073fcf..2a5eb46 100644 --- a/lib/local/Recorder/include/RecorderOpenFaceParameters.h +++ b/lib/local/Utilities/include/RecorderOpenFaceParameters.h @@ -44,7 +44,7 @@ using namespace std; -namespace Recorder +namespace Utilities { class RecorderOpenFaceParameters diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h new file mode 100644 index 0000000..ffe61e5 --- /dev/null +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -0,0 +1,100 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __SEQUENCE_CAPTURE_h_ +#define __SEQUENCE_CAPTURE_h_ + +// System includes +#include +#include +#include + +// OpenCV includes +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for capturing sequences from video, webcam, and image directories + */ + class SequenceCapture { + + public: + + // Default constructor + SequenceCapture(); + + // Opening based on command line arguments + void Open(std::vector arguments); + + // Direct opening + + // Webcam + void OpenWebcam(int device_id); + + // Image sequence in the directory + void OpenImageSequence(std::string directory); + + // Video file + void OpenVideoFile(std::string video_file); + + private: + + // Used for capturing webcam and video + cv::VideoCapture capture; + + // Storing the latest captures + cv::Mat latest_frame; + cv::Mat latest_gray_frame; + + double fps; + + // Keeping track if we are opening a video, webcam or image sequence + bool is_webcam; + bool is_image_seq; + + // Keeping track of frame number and the files in the image sequence + int frame_num; + std::vector image_files; + + // Length of video allowing to assess progress + int vid_length; + + // TODO fx and fy should be here + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Recorder/src/RecorderCSV.cpp b/lib/local/Utilities/src/RecorderCSV.cpp similarity index 99% rename from lib/local/Recorder/src/RecorderCSV.cpp rename to lib/local/Utilities/src/RecorderCSV.cpp index e8ef507..fb7e454 100644 --- a/lib/local/Recorder/src/RecorderCSV.cpp +++ b/lib/local/Utilities/src/RecorderCSV.cpp @@ -39,7 +39,7 @@ // For standard out #include -using namespace Recorder; +using namespace Utilities; // Default constructor initializes the variables RecorderCSV::RecorderCSV():output_file(){}; diff --git a/lib/local/Recorder/src/RecorderHOG.cpp b/lib/local/Utilities/src/RecorderHOG.cpp similarity index 99% rename from lib/local/Recorder/src/RecorderHOG.cpp rename to lib/local/Utilities/src/RecorderHOG.cpp index ac5563f..ea0a218 100644 --- a/lib/local/Recorder/src/RecorderHOG.cpp +++ b/lib/local/Utilities/src/RecorderHOG.cpp @@ -35,7 +35,7 @@ #include -using namespace Recorder; +using namespace Utilities; // Default constructor initializes the variables RecorderHOG::RecorderHOG() :hog_file() {}; diff --git a/lib/local/Recorder/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp similarity index 99% rename from lib/local/Recorder/src/RecorderOpenFace.cpp rename to lib/local/Utilities/src/RecorderOpenFace.cpp index 76d708c..1d5a791 100644 --- a/lib/local/Recorder/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -48,7 +48,7 @@ using namespace boost::filesystem; -using namespace Recorder; +using namespace Utilities; #define WARN_STREAM( stream ) \ std::cout << "Warning: " << stream << std::endl diff --git a/lib/local/Recorder/src/RecorderOpenFaceParameters.cpp b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp similarity index 99% rename from lib/local/Recorder/src/RecorderOpenFaceParameters.cpp rename to lib/local/Utilities/src/RecorderOpenFaceParameters.cpp index fc3a818..a563f7a 100644 --- a/lib/local/Recorder/src/RecorderOpenFaceParameters.cpp +++ b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp @@ -35,7 +35,7 @@ using namespace std; -using namespace Recorder; +using namespace Utilities; RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector &arguments, bool sequence, double fps_vid_out) { diff --git a/matlab_runners/Feature Point Experiments/run_yt_dataset.m b/matlab_runners/Feature Point Experiments/run_yt_dataset.m index 8d3c29a..35738ff 100644 --- a/matlab_runners/Feature Point Experiments/run_yt_dataset.m +++ b/matlab_runners/Feature Point Experiments/run_yt_dataset.m @@ -27,18 +27,14 @@ database_root = [database_root, '/ytceleb/']; in_vids = dir([database_root '/*.avi']); command = executable; -command = cat(2, command, ' -no3Dfp -noMparams -noPose -noGaze -noAUs '); +command = cat(2, command, ' -2Dfp '); % add all videos to single argument list (so as not to load the model anew % for every video) for i=1:numel(in_vids) - [~, name, ~] = fileparts(in_vids(i).name); - - % where to output tracking results - outputFile_fp = [output name '_fp.txt']; in_file_name = [database_root, '/', in_vids(i).name]; - command = cat(2, command, [' -f "' in_file_name '" -of "' outputFile_fp '"']); + command = cat(2, command, [' -f "' in_file_name '" -of "' output '"']); end if(isunix) @@ -56,19 +52,13 @@ end command = executable; command = cat(2, command, ' -mloc model/main_clm_general.txt '); -command = cat(2, command, ' -no3Dfp -noMparams -noPose -noGaze -noAUs '); +command = cat(2, command, ' -2Dfp '); % add all videos to single argument list (so as not to load the model anew % for every video) -for i=1:numel(in_vids) - - [~, name, ~] = fileparts(in_vids(i).name); - - % where to output tracking results - outputFile_fp = [output name '_fp.txt']; - in_file_name = [database_root, '/', in_vids(i).name]; - - command = cat(2, command, [' -f "' in_file_name '" -of "' outputFile_fp '"']); +for i=1:numel(in_vids) + in_file_name = [database_root, '/', in_vids(i).name]; + command = cat(2, command, [' -f "' in_file_name '" -of "' output '"']); end if(isunix) @@ -80,7 +70,7 @@ end d_loc = 'yt_features/'; d_loc_clm = 'yt_features_clm/'; -files_yt = dir([d_loc, '/*.txt']); +files_yt = dir([d_loc, '/*.csv']); preds_all = []; preds_all_clm = []; gts_all = [];