diff --git a/lib/3rdParty/OpenCV3.4/openCV3.4.props b/lib/3rdParty/OpenCV3.4/openCV3.4.props index c0c7498..f604b26 100644 --- a/lib/3rdParty/OpenCV3.4/openCV3.4.props +++ b/lib/3rdParty/OpenCV3.4/openCV3.4.props @@ -8,8 +8,8 @@ $(SolutionDir)lib\3rdParty\OpenCV3.4\include\opencv;$(SolutionDir)lib\3rdParty\OpenCV3.4\include;%(AdditionalIncludeDirectories) - $(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib;%(AdditionalLibraryDirectories) - opencv_world340.lib;opencv_world340d.lib;%(AdditionalDependencies) + $(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib\$(Configuration);%(AdditionalLibraryDirectories) + opencv_world340.lib;%(AdditionalDependencies) xcopy /I /E /Y /D /C "$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\bin\$(Configuration)" "$(OutDir)" diff --git a/lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340d.lib b/lib/3rdParty/OpenCV3.4/x64/v140/lib/Debug/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340d.lib rename to lib/3rdParty/OpenCV3.4/x64/v140/lib/Debug/opencv_world340.lib diff --git a/lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340.lib b/lib/3rdParty/OpenCV3.4/x64/v140/lib/Release/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340.lib rename to lib/3rdParty/OpenCV3.4/x64/v140/lib/Release/opencv_world340.lib diff --git a/lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340d.lib b/lib/3rdParty/OpenCV3.4/x86/v140/lib/Debug/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340d.lib rename to lib/3rdParty/OpenCV3.4/x86/v140/lib/Debug/opencv_world340.lib diff --git a/lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340.lib b/lib/3rdParty/OpenCV3.4/x86/v140/lib/Release/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340.lib rename to lib/3rdParty/OpenCV3.4/x86/v140/lib/Release/opencv_world340.lib diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index 118a89e..1a775a3 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -226,6 +226,7 @@ private: int align_width_au; int align_height_au; + bool align_mask; double align_scale_out; int align_width_out; int align_height_out; diff --git a/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h b/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h index 5473225..2bc29b6 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h @@ -64,11 +64,12 @@ public: bool grayscale; // Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense - void setAlignedOutput(int output_size, double scale=-1); + void setAlignedOutput(int output_size, double scale=-1, bool masked = true); // This will also change the model location void OptimizeForVideos(); void OptimizeForImages(); + double getAlignMask() const { return sim_align_face_mask; } double getSimScaleOut() const { return sim_scale_out; } int getSimSizeOut() const { return sim_size_out; } bool getDynamic() const { return dynamic; } @@ -83,6 +84,9 @@ private: double sim_scale_out; int sim_size_out; + // Should aligned face be masked out from background + bool sim_align_face_mask; + // Should a video stream be assumed bool dynamic; diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index f1fc511..1d579fc 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -63,6 +63,7 @@ FaceAnalyser::FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_anal { this->Read(face_analyser_params.getModelLoc()); + align_mask = face_analyser_params.getAlignMask(); align_scale_out = face_analyser_params.getSimScaleOut(); align_width_out = face_analyser_params.getSimSizeOut(); align_height_out = face_analyser_params.getSimSizeOut(); @@ -256,17 +257,24 @@ void FaceAnalyser::PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, cons cv::Mat_ params_local; pdm.CalcParams(params_global, params_local, detected_landmarks); - // First align the face - AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, 0.7, 112, 112); - - // If the output requirement matches use the already computed one, else compute it again - if (align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au) + // The aligned face requirement for AUs + AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au); + + // If the aligned face for AU matches the output requested one, just reuse it, else compute it + if (align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au && align_mask) { aligned_face_for_output = aligned_face_for_au.clone(); } else { - AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out); + if (align_mask) + { + AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out); + } + else + { + AlignFace(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, true, align_scale_out, align_width_out, align_height_out); + } } // Extract HOG descriptor from the frame and convert it to a useable format @@ -315,6 +323,7 @@ void FaceAnalyser::PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, cons } + void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online) { @@ -333,14 +342,21 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& det // The aligned face requirement for AUs AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au); - // If the output requirement matches use the already computed one, else compute it again - if(align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au) + // If the aligned face for AU matches the output requested one, just reuse it, else compute it + if (align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au && align_mask) { aligned_face_for_output = aligned_face_for_au.clone(); } else { - AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out); + if (align_mask) + { + AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out); + } + else + { + AlignFace(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, true, align_scale_out, align_width_out, align_height_out); + } } } else diff --git a/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp index 05ef007..9c4db3b 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp @@ -88,6 +88,11 @@ FaceAnalyserParameters::FaceAnalyserParameters(vector &arguments):root() grayscale = true; valid[i] = false; } + else if (arguments[i].compare("-nomask") == 0) + { + sim_align_face_mask = false; + valid[i] = false; + } else if (arguments[i].compare("-simscale") == 0) { sim_scale_out = stod(arguments[i + 1]); @@ -155,6 +160,7 @@ void FaceAnalyserParameters::init() this->grayscale = false; this->sim_scale_out = 0.7; this->sim_size_out = 112; + this->sim_align_face_mask = true; this->model_location = "AU_predictors/main_dynamic_svms.txt"; @@ -184,13 +190,15 @@ void FaceAnalyserParameters::init() } // Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense -void FaceAnalyserParameters::setAlignedOutput(int output_size, double scale) +void FaceAnalyserParameters::setAlignedOutput(int output_size, double scale, bool masked) { this->sim_size_out = output_size; // If we set the size but not the scale, adapt the scale to the right size if (scale ==-1) this->sim_scale_out = sim_size_out * (0.7 / 112.0); else this->sim_scale_out = sim_scale_out; + this->sim_align_face_mask = masked; + } // This will also change the model location void FaceAnalyserParameters::OptimizeForVideos() diff --git a/lib/local/FaceAnalyser/src/Face_utils.cpp b/lib/local/FaceAnalyser/src/Face_utils.cpp index f2cfe25..334984e 100644 --- a/lib/local/FaceAnalyser/src/Face_utils.cpp +++ b/lib/local/FaceAnalyser/src/Face_utils.cpp @@ -114,7 +114,7 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid, float sim_scale, int out_width, int out_height) + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6f params_global, const PDM& pdm, bool rigid, float sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; @@ -131,19 +131,18 @@ namespace FaceAnalysis extract_rigid_points(source_landmarks, destination_landmarks); } - // TODO rem the doubles here - cv::Matx22d scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); - cv::Matx23d warp_matrix; + cv::Matx22f scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); + cv::Matx23f warp_matrix; warp_matrix(0,0) = scale_rot_matrix(0,0); warp_matrix(0,1) = scale_rot_matrix(0,1); warp_matrix(1,0) = scale_rot_matrix(1,0); warp_matrix(1,1) = scale_rot_matrix(1,1); - double tx = params_global[4]; - double ty = params_global[5]; + float tx = params_global[4]; + float ty = params_global[5]; - cv::Vec2d T(tx, ty); + cv::Vec2f T(tx, ty); T = scale_rot_matrix * T; // Make sure centering is correct